diff --git a/.gitignore b/.gitignore index c4bed011dfb..a4343946b89 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ Makefile.in *.rej .deps .libs +*~ /configure /config.cache @@ -54,6 +55,7 @@ doc/man/doveadm-exec.1 doc/man/doveadm-expunge.1 doc/man/doveadm-fetch.1 doc/man/doveadm-flags.1 +doc/man/doveadm-fs.1 doc/man/doveadm-fts.1 doc/man/doveadm-import.1 doc/man/doveadm-instance.1 diff --git a/Makefile.am b/Makefile.am index a7be896d2fd..bc49ef8acde 100644 --- a/Makefile.am +++ b/Makefile.am @@ -14,6 +14,7 @@ EXTRA_DIST = \ ChangeLog \ cc-wrapper.sh.in \ update-version.sh \ + run-test-valgrind.supp \ $(conf_DATA) noinst_DATA = dovecot-config diff --git a/NEWS b/NEWS index 52315d15681..4f72c0ba2fe 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,139 @@ +v2.2.25 2016-06-xx Timo Sirainen + + * lmtp: Start tracking lmtp_user_concurrency_limit and reject already + at RCPT TO stage. This avoids MTA unnecessarily completing DATA only + to get an error. + * doveadm: Previously only mail settings were read from protocol + doveadm { .. } section. Now all settings are. + + + quota: Added quota_over_flag_lazy_check setting. It avoids checking + quota_over_flag always at startup. Instead it's checked only when + quota is being read for some other purpose. + + auth: Added a new auth policy service: + http://wiki2.dovecot.org/Authentication/Policy + + auth: Added PBKDF2 password scheme + + auth: Added %{auth_user}, %{auth_username} and %{auth_domain} + + auth: Added ":remove" suffix to extra field names to remove them. + + auth: Added "delay_until=[+]" passdb + extra field. The auth will wait until and optionally some + randomness and then return success. + + dict proxy: Added idle_msecs= parameter. Support async operations. + + Performance improvements for handling large mailboxes. + + Added lib-dcrypt API for providing cryptographic functions. + + Added "doveadm mailbox update" command + + imap commands' output now includes timing spent on the "syncing" + stage if it's larger than 0. + + cassandra: Added metrics= to connect setting to output internal + statistics in JSON format every second to . + + doveadm mailbox delete: Added -e parameter to delete only empty + mailboxes. Added --unsafe option to quickly delete a mailbox, + bypassing lazy_expunge and quota plugins. + + doveadm user & auth cache flush are now available via doveadm-server. + + doveadm service stop will stop specified services while + leaving the rest of Dovecot running. + + quota optimization: Avoid reading mail sizes for backends which + don't need them (count, fs, dirsize) + + Added mailbox { autoexpunge_max_mails= } setting. + + Added welcome plugin: http://wiki2.dovecot.org/Plugins/Welcome + + fts: Added fts_autoindex_exclude setting. + - v2.2.24's MIME parser was assert-crashing on mails having truncated + MIME headers. + - auth: With multiple userdbs the final success/failure result wasn't + always correct. The last userdb's result was always used. + - doveadm backup was sometimes deleting entire mailboxes unnecessarily. + - doveadm: Command -parameters weren't being sent to doveadm-server. + - If dovecot.index read failed e.g. because mmap() reached VSZ limit, + an empty index could have been opened instead, corrupting the + mailbox state. + - imapc: Fixed EXPUNGE handling when imapc_features didn't have modseq. + - lazy-expunge: Fixed a crash when copying failed. Various other fixes. + - fts-lucene: Fixed crash on index rescan. + - auth_stats=yes produced broken output + - dict-ldap: Various fixes + - dict-sql: NULL values crashed. Now they're treated as "not found". + +v2.2.24 2016-04-26 Timo Sirainen + + * doveconf now warns if it sees a global setting being changed when + the same setting was already set inside some filters. (A common + mistake has been adding more plugins to a global mail_plugins + setting after it was already set inside protocol { .. }, which + caused the global setting to be ignored for that protocol.) + * LMTP proxy: Increased default timeout 30s -> 125s. This makes it + less likely to reach the timeout and cause duplicate deliveries. + * LMTP and indexer now append ":suffix" to session IDs to make it + unique for the specific user's delivery. (Fixes duplicate session + ID warnings in stats process.) + + + Added dict-ldap for performing read-only LDAP dict lookups. + + lazy-expunge: All mails can be saved to a single specified mailbox. + + mailbox { autoexpunge } supports now wildcards in mailbox names. + + doveadm HTTP API: Added support for proxy commands + + imapc: Reconnect when getting disconnected in non-selected state. + + imapc: Added imapc_features=modseq to access MODSEQs/HIGHESTMODSEQ. + This is especially useful for incremental dsync. + + doveadm auth/user: Auth lookup performs debug logging if + -o auth_debug=yes is given to doveadm. + + Added passdb/userdb { auth_verbose=yes|no } setting. + + Cassandra: Added user, password, num_threads, connect_timeout and + request_timeout settings. + + doveadm user -e : Print with %variables expanded. + - Huge header lines could have caused Dovecot to use too much memory + (depending on config and used IMAP commands). (Typically this would + result in only the single user's process dying with out of memory + due to reaching service { vsz_limit } - not a global DoS). + - dsync: Detect and handle invalid/stale -s state string better. + - dsync: Fixed crash caused by specific mailbox renames + - auth: Auth cache is now disabled passwd-file. It was unnecessary and + it broke %variables in extra fields. + - fts-tika: Don't crash if it returns 500 error + - dict-redis: Fixed timeout handling + - SEARCH INTHREAD was crashing + - stats: Only a single fifo_listeners was supported, making it + impossible to use both auth_stats=yes and mail stats plugin. + - SSL errors were logged in separate "Stacked error" log lines + instead of as part of the disconnection reason. + - MIME body parser didn't handle properly when a child MIME part's + --boundary had the same prefix as the parent. + +v2.2.23 2016-03-30 Timo Sirainen + + - Various fixes to doveadm. Especially running commands via + doveadm-server was broken. + - director: Fixed user weakness getting stuck in some situations + - director: Fixed a situation where directors keep re-sending + different states to each others and never becoming synced. + - director: Fixed assert-crash related to a slow "user killed" reply + - Fixed assert-crash related to istream-concat, which could have + been triggered at least by a Sieve script. + +v2.2.22 2016-03-16 Timo Sirainen + + + Added doveadm HTTP API: See + http://wiki2.dovecot.org/Design/DoveadmProtocol/HTTP + + virtual plugin: Mailbox filtering can now be done based on the + mailbox metadata. See http://wiki2.dovecot.org/Plugins/Virtual + + stats: Added doveadm stats reset to reset global stats. + + stats: Added authentication statistics if auth_stats=yes. + + dsync, imapc, pop3c & pop3-migration: Many optimizations, + improvements and error handling fixes. + + doveadm: Most commands now stop soon after SIGINT/SIGTERM. + - auth: Auth caching was done too aggressively when %variables were + used in default_fields, override_fields or LDAP pass/user_attrs. + userdb result_* were also ignored when user was found from cache. + - imap: Fixed various assert-crashes caused v2.2.20+. Some of them + caught actual hangs or otherwise unwanted behavior towards IMAP + clients. + - Expunges were forgotten in some situations, for example when + pipelining multiple IMAP MOVE commands. + - quota: Per-namespaces quota were broken for dict and count backends + in v2.2.20+ + - fts-solr: Search queries were using OR instead of AND as the + separator for multi-token search queries in v2.2.20+. + - Single instance storage support wasn't really working in v2.2.16+ + - dbox: POP3 message ordering wasn't working correctly. + - virtual plugin: Fixed crashes related to backend mailbox deletions. + v2.2.21 2015-12-11 Timo Sirainen - doveadm mailbox list (and some others) were broken in v2.2.20 diff --git a/README b/README index 109ba8f7533..32fe185f16b 100644 --- a/README +++ b/README @@ -63,13 +63,18 @@ IMAP extensions: 5256 - IMAP SORT and THREAD Extensions 5258 - IMAP4 - LIST Command Extensions 5267 - Contexts for IMAP4 + 5464 - The IMAP METADATA Extension + 5465 - The IMAP NOTIFY Extension 5524 - Extended URLFETCH for Binary and Converted Parts 5530 - IMAP Response Codes 5819 - IMAP4 Extension for Returning STATUS Information in Extended LIST 5957 - Display-Based Address Sorting for the IMAP4 SORT Extension 6154 - IMAP LIST Extension for Special-Use Mailboxes 6203 - IMAP4 Extension for Fuzzy Search + 6785 - Support for IMAP Events in Sieve (via Pigeonhole plugin) 6851 - Internet Message Access Protocol (IMAP) - MOVE Extension + 7162 - IMAP Extensions: Quick Flag Changes Resynchronization (CONDSTORE) + and Quick Mailbox Resynchronization (QRESYNC) Contact info ------------ diff --git a/configure.ac b/configure.ac index 8d7445fefe1..2ebc28e5ad0 100644 --- a/configure.ac +++ b/configure.ac @@ -2,8 +2,8 @@ AC_PREREQ([2.59]) # Be sure to update ABI version also if anything changes that might require # recompiling plugins. Most importantly that means if any structs are changed. -AC_INIT([Dovecot],[2.2.devel],[dovecot@dovecot.org]) -AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv20($PACKAGE_VERSION)", [Dovecot ABI version]) +AC_INIT([Dovecot],[2.2.25.rc1],[dovecot@dovecot.org]) +AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv25($PACKAGE_VERSION)", [Dovecot ABI version]) AC_CONFIG_SRCDIR([src]) @@ -456,7 +456,7 @@ AC_CHECK_FUNCS(fcntl flock lockf inet_aton sigaction getpagesize madvise \ getmntinfo setpriority quotactl getmntent kqueue kevent \ backtrace_symbols walkcontext dirfd clearenv \ malloc_usable_size glob fallocate posix_fadvise \ - getpeereid getpeerucred inotify_init) + getpeereid getpeerucred inotify_init timegm) AC_CHECK_TYPES([struct sockpeercred],,,[ #include @@ -1007,6 +1007,24 @@ if test $i_cv_signed_time_t = yes; then AC_DEFINE(TIME_T_SIGNED,, [Define if your time_t is signed]) fi +AC_CACHE_CHECK([if we can use C99 static in array sizes],i_cv_c99_static_arrays,[ + AC_TRY_COMPILE([ + void foo(int arr[static 20]); + ], [ + ], [ + i_cv_c99_static_arrays=yes + ], [ + i_cv_c99_static_arrays=no + ]) +]) + +if test $i_cv_c99_static_arrays = yes; then + static_value=static +else + static_value= +fi +AC_DEFINE_UNQUOTED(STATIC_ARRAY, $static_value, [C99 static array]) + dnl Our implementation of AC_C_FLEXIBLE_ARRAY_MEMBER. dnl Use it until autoconf 2.61+ becomes more widely used AC_CACHE_CHECK([if we can use C99-like flexible array members],i_cv_c99_flex_arrays,[ @@ -1692,6 +1710,9 @@ if test $want_openssl != no && test $have_ssl = no; then AC_CHECK_LIB(ssl, SSL_get_servername, [ AC_DEFINE(HAVE_SSL_GET_SERVERNAME,, [Build with TLS hostname support]) ],, $SSL_LIBS) + AC_CHECK_LIB(ssl, SSL_COMP_free_compression_methods, [ + AC_DEFINE(HAVE_SSL_COMP_FREE_COMPRESSION_METHODS,, [Build with SSL_COMP_free_compression_methods() support]) + ],, $SSL_LIBS) fi fi AM_CONDITIONAL(BUILD_OPENSSL, test "$have_openssl" = "yes") @@ -2044,7 +2065,7 @@ else fi fi AM_CONDITIONAL(LDAP_PLUGIN, test "$have_ldap_plugin" = "yes") -AM_CONDITIONAL(HAVE_LDAP, test "$want_ldap" = "yes") +AM_CONDITIONAL(HAVE_LDAP, test "$want_ldap" != "no") dict_drivers= if test $want_db != no; then @@ -2516,6 +2537,7 @@ dnl ** dnl ** Shared libraries usage dnl ** +LIBDOVECOT_LA_LIBS='$(top_builddir)/src/lib-dict-extra/libdict_extra.la $(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-stats/libstats.la $(top_builddir)/src/lib-http/libhttp.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-sasl/libsasl.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la $(top_builddir)/src/lib-dcrypt/libdcrypt.la $(top_builddir)/src/lib-test/libtest.la $(top_builddir)/src/lib/liblib.la' if test "$want_shared_libs" = "yes"; then LIBDOVECOT_DEPS='$(top_builddir)/src/lib-dovecot/libdovecot.la' LIBDOVECOT="$LIBDOVECOT_DEPS \$(MODULE_LIBS)" @@ -2523,13 +2545,13 @@ if test "$want_shared_libs" = "yes"; then LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/libdovecot-login.la' LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/libdovecot-lda.la' else - LIBDOVECOT_DEPS='$(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-stats/libstats.la $(top_builddir)/src/lib-http/libhttp.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-sasl/libsasl.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la $(top_builddir)/src/lib-test/libtest.la $(top_builddir)/src/lib/liblib.la' + LIBDOVECOT_DEPS="$LIBDOVECOT_LA_LIBS" LIBDOVECOT="$LIBDOVECOT_DEPS \$(LIBICONV) \$(MODULE_LIBS)" LIBDOVECOT_STORAGE_DEPS='$(top_builddir)/src/lib-storage/libstorage.la' LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/liblogin.la' LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/liblda.la' fi -if test $want_ldap = yes; then +if test $want_ldap != no; then LIBDOVECOT_LDAP='$(top_builddir)/src/lib-ldap/libdovecot-ldap.la' else LIBDOVECOT_LDAP='' @@ -2540,6 +2562,7 @@ LIBDOVECOT_SQL='$(top_builddir)/src/lib-sql/libsql.la' LIBDOVECOT_COMPRESS='$(top_builddir)/src/lib-compression/libcompression.la' LIBDOVECOT_LIBFTS='$(top_builddir)/src/lib-fts/libfts.la' AC_SUBST(LIBDOVECOT) +AC_SUBST(LIBDOVECOT_LA_LIBS) AC_SUBST(LIBDOVECOT_DEPS) AC_SUBST(LIBDOVECOT_STORAGE) AC_SUBST(LIBDOVECOT_STORAGE_DEPS) @@ -2858,7 +2881,9 @@ src/lib-sql/Makefile src/lib-auth/Makefile src/lib-charset/Makefile src/lib-compression/Makefile +src/lib-dcrypt/Makefile src/lib-dict/Makefile +src/lib-dict-extra/Makefile src/lib-dns/Makefile src/lib-fs/Makefile src/lib-fts/Makefile @@ -2925,6 +2950,7 @@ src/plugins/Makefile src/plugins/acl/Makefile src/plugins/imap-acl/Makefile src/plugins/autocreate/Makefile +src/plugins/dict-ldap/Makefile src/plugins/expire/Makefile src/plugins/fs-compress/Makefile src/plugins/fts/Makefile @@ -2949,6 +2975,7 @@ src/plugins/stats/Makefile src/plugins/imap-stats/Makefile src/plugins/trash/Makefile src/plugins/virtual/Makefile +src/plugins/welcome/Makefile src/plugins/zlib/Makefile src/plugins/imap-zlib/Makefile stamp.h diff --git a/doc/man/Makefile.am b/doc/man/Makefile.am index 3bfaea3d231..6390ed769ba 100644 --- a/doc/man/Makefile.am +++ b/doc/man/Makefile.am @@ -27,6 +27,7 @@ nodist_man1_MANS = \ doveadm-expunge.1 \ doveadm-fetch.1 \ doveadm-flags.1 \ + doveadm-fs.1 \ doveadm-fts.1 \ doveadm-import.1 \ doveadm-instance.1 \ @@ -75,6 +76,7 @@ EXTRA_DIST = \ doveadm-expunge.1.in \ doveadm-fetch.1.in \ doveadm-flags.1.in \ + doveadm-fs.1.in \ doveadm-fts.1.in \ doveadm-import.1.in \ doveadm-instance.1.in \ diff --git a/doc/man/doveadm-expunge.1.in b/doc/man/doveadm-expunge.1.in index 14b0f4318e6..41d7571775f 100644 --- a/doc/man/doveadm-expunge.1.in +++ b/doc/man/doveadm-expunge.1.in @@ -39,7 +39,7 @@ commands. .PP In the first form, .BR doveadm (1) -will executed the +will execute the .B expunge action with the environment of the logged in system user. .PP diff --git a/doc/man/doveadm-fs.1.in b/doc/man/doveadm-fs.1.in new file mode 100644 index 00000000000..0c30f7d2f4b --- /dev/null +++ b/doc/man/doveadm-fs.1.in @@ -0,0 +1,93 @@ +.\" Copyright (c) 2016 Dovecot authors, see the included COPYING file +.TH DOVEADM\-PROXY 1 "2016-04-26" "Dovecot v2.2" "Dovecot" +.SH NAME +doveadm\-fs \- Interact with the abstract mail storage filesystem +.\"------------------------------------------------------------------------ +.SH SYNOPSIS +.BR doveadm " [" \-Dv ] +[\fB\-f\fP \fIformatter\fP] +.BI fs \ command +.RI [ ARGUMENTS ] +.\"------------------------------------------------------------------------ +.SH DESCRIPTION +The +.B doveadm fs +.I commands +are used to abstractly interact with the storage backend defined in the +Dovecot configuration. It allows access to the mailbox structure without +needing to know details of how the storage backend is architected. +.\"------------------------------------------------------------------------ +@INCLUDE:global-options-formatter@ +.\" --- command specific options --- "/. +.PP +This command uses by default the +.B table +output formatter. +.\"------------------------------------------------------------------------ +.SH COMMANDS +.SS fs copy +.B doveadm fs copy +.I fs-driver +.I fs-args +.I source-path +.I dest-path +.PP +Copy source path to the destination path. +.\"------------------------------------- +.SS fs delete +.B doveadm fs delete +[\fB\-R\fP] [\fB\-n\fP \fIcount\fP] +.I fs-driver +.I fs-args +.I path +[\fIpath\fP ...] +.PP +Delete all data associated with the path provided. +.\"------------------------------------- +.SS fs get +.B doveadm fs get +.I fs-driver +.I fs-args +.I path +.PP +Retrieve data associated with the path provided. +.\"------------------------------------- +.SS fs iter +.B doveadm fs iter +.I fs-driver +.I fs-args +.I path +.PP +Iterate through all data files in the path provided. +.\"------------------------------------- +.SS fs iter-dirs +.B doveadm fs iter-dirs +.I fs-driver +.I fs-args +.I path +.PP +Iterate through all directories in the path provided. +.\"------------------------------------- +.SS fs put +.B doveadm fs put +[\fB\-h\fP \fIhash\fP] +.I fs-driver +.I fs-args +.I input_path +.I path +.PP +Store data at the path provided. +.\"------------------------------------- +.SS fs stat +.B doveadm fs stat +.I fs-driver +.I fs-args +.I path +.PP +Retrieve files status for the path provided. Currently, only the total size +(in bytes) of the item is returned. +.\"------------------------------------------------------------------------ +@INCLUDE:reporting-bugs@ +.\"------------------------------------------------------------------------ +.SH SEE ALSO +.BR doveadm (1) diff --git a/doc/man/doveadm-sync.1.in b/doc/man/doveadm-sync.1.in index 2d56391e63c..7b69bbe599c 100644 --- a/doc/man/doveadm-sync.1.in +++ b/doc/man/doveadm-sync.1.in @@ -74,6 +74,33 @@ multiple times, then switch mails to be delivered to the new mailbox and run .B doveadm sync \-1 once more to transfer any last new mails from the old mailbox. +.IP +The one\-way algorithm is the same as two-way dsync algorithm except the +source account is not modified. It fetches the message's GUID (Global UID), +which is used to identify any conflicting UIDs in messages. As long as the +source and destination side has matching UID<\->GUID mapping, those emails +are assumed to be synced correctly. Only after the first mismatch will +changes begin. +.IP +Example: Source mailbox has messages UID 1..5; source mailbox is sync'd +using +.B doveadm backup +to the destination. Subsequently, UID 6 is delivered to the source mailbox +and UID 1 is expunged from the destination mailbox. In this example, UID 1 +is kept removed (in destination) because UID 1..5 have identical +Date+Message\-ID headers. UID 6 is not seen in destination so it's copied. +.IP +If both source and destination have UID 6, but the messages are different, +the headers don't match and both the messages are kept in the destination but +they're given new UIDs 7 and 8 just to be sure any client didn't get confused +about what UID 11 actually was. Thus, one\-way sync begins to quickly diverge +from the source mailbox once changes start to occur on either side; one\-way +sync should therefore normally only be used within a short period of time +after a +.B doveadm backup +or +.B doveadm sync +command was used to synchronize the mailboxes. .\"------------------------------------- .RE .PP diff --git a/dovecot-config.in.in b/dovecot-config.in.in index c71bf52d5d2..f776c387cf2 100644 --- a/dovecot-config.in.in +++ b/dovecot-config.in.in @@ -22,13 +22,14 @@ LIBDOVECOT_STORAGE_DEPS="@LIBDOVECOT_STORAGE_DEPS@" LIBDOVECOT_DSYNC_DEPS="@LIBDOVECOT_DSYNC@" LIBDOVECOT_LIBFTS_DEPS="@LIBDOVECOT_LIBFTS@" -LIBDOVECOT_INCLUDE="-I$(incdir) -I$(incdir)/src/lib -I$(incdir)/src/lib-dict -I$(incdir)/src/lib-dns -I$(incdir)/src/lib-http -I$(incdir)/src/lib-mail -I$(incdir)/src/lib-imap -I$(incdir)/src/lib-fs -I$(incdir)/src/lib-charset -I$(incdir)/src/lib-auth -I$(incdir)/src/lib-master -I$(incdir)/src/lib-ssl-iostream -I$(incdir)/src/lib-compression -I$(incdir)/src/lib-settings -I$(incdir)/src/lib-test -I$(incdir)/src/lib-sasl -I$(incdir)/src/lib-stats" +LIBDOVECOT_INCLUDE="-I$(incdir) -I$(incdir)/src/lib -I$(incdir)/src/lib-dict -I$(incdir)/src/lib-dns -I$(incdir)/src/lib-http -I$(incdir)/src/lib-mail -I$(incdir)/src/lib-imap -I$(incdir)/src/lib-fs -I$(incdir)/src/lib-charset -I$(incdir)/src/lib-auth -I$(incdir)/src/lib-master -I$(incdir)/src/lib-ssl-iostream -I$(incdir)/src/lib-compression -I$(incdir)/src/lib-settings -I$(incdir)/src/lib-test -I$(incdir)/src/lib-sasl -I$(incdir)/src/lib-stats -I$(incdir)/src/lib-dcrypt" LIBDOVECOT_LDA_INCLUDE="-I$(incdir)/src/lib-lda -I$(incdir)/src/lda" LIBDOVECOT_AUTH_INCLUDE="-I$(incdir)/src/auth" LIBDOVECOT_DOVEADM_INCLUDE="-I$(incdir)/src/doveadm" LIBDOVECOT_STORAGE_INCLUDE="-I$(incdir)/src/lib-index -I$(incdir)/src/lib-storage -I$(incdir)/src/lib-storage/list -I$(incdir)/src/lib-storage/index -I$(incdir)/src/lib-storage/index/raw -I$(incdir)/src/lib-imap-storage -I$(incdir)/src/plugins/quota" LIBDOVECOT_DSYNC_INCLUDE="-I$(incdir)/src/doveadm/dsync" LIBDOVECOT_LOGIN_INCLUDE="-I$(incdir)/src/login-common" +LIBDOVECOT_SQL_INCLUDE="-I$(incdir)/src/lib-sql" LIBDOVECOT_IMAP_LOGIN_INCLUDE="-I$(incdir)/src/imap-login" LIBDOVECOT_IMAP_INCLUDE="-I$(incdir)/src/imap" LIBDOVECOT_POP3_INCLUDE="-I$(incdir)/src/pop3" diff --git a/dovecot.m4 b/dovecot.m4 index 30b9ae258fb..efecf1c3bb3 100644 --- a/dovecot.m4 +++ b/dovecot.m4 @@ -6,7 +6,7 @@ # unlimited permission to copy and/or distribute it, with or without # modifications, as long as this notice is preserved. -# serial 21 +# serial 22 AC_DEFUN([DC_DOVECOT_MODULEDIR],[ AC_ARG_WITH(moduledir, @@ -128,7 +128,7 @@ AC_DEFUN([DC_DOVECOT],[ AX_SUBST_L([DOVECOT_INSTALLED], [DOVECOT_CFLAGS], [DOVECOT_LIBS], [DOVECOT_SSL_LIBS], [DOVECOT_SQL_LIBS], [DOVECOT_COMPRESS_LIBS]) AX_SUBST_L([LIBDOVECOT], [LIBDOVECOT_LOGIN], [LIBDOVECOT_SQL], [LIBDOVECOT_SSL], [LIBDOVECOT_COMPRESS], [LIBDOVECOT_LDA], [LIBDOVECOT_STORAGE], [LIBDOVECOT_DSYNC], [LIBDOVECOT_LIBFTS]) AX_SUBST_L([LIBDOVECOT_DEPS], [LIBDOVECOT_LOGIN_DEPS], [LIBDOVECOT_SQL_DEPS], [LIBDOVECOT_SSL_DEPS], [LIBDOVECOT_COMPRESS_DEPS], [LIBDOVECOT_LDA_DEPS], [LIBDOVECOT_STORAGE_DEPS], [LIBDOVECOT_DSYNC_DEPS], [LIBDOVECOT_LIBFTS_DEPS]) - AX_SUBST_L([LIBDOVECOT_INCLUDE], [LIBDOVECOT_LDA_INCLUDE], [LIBDOVECOT_AUTH_INCLUDE], [LIBDOVECOT_DOVEADM_INCLUDE], [LIBDOVECOT_SERVICE_INCLUDE], [LIBDOVECOT_STORAGE_INCLUDE], [LIBDOVECOT_LOGIN_INCLUDE], [LIBDOVECOT_IMAP_LOGIN_INCLUDE], [LIBDOVECOT_CONFIG_INCLUDE], [LIBDOVECOT_IMAP_INCLUDE], [LIBDOVECOT_POP3_INCLUDE], [LIBDOVECOT_DSYNC_INCLUDE], [LIBDOVECOT_IMAPC_INCLUDE], [LIBDOVECOT_FTS_INCLUDE], [LIBDOVECOT_NOTIFY_INCLUDE], [LIBDOVECOT_ACL_INCLUDE], [LIBDOVECOT_LIBFTS_INCLUDE]) + AX_SUBST_L([LIBDOVECOT_INCLUDE], [LIBDOVECOT_LDA_INCLUDE], [LIBDOVECOT_AUTH_INCLUDE], [LIBDOVECOT_DOVEADM_INCLUDE], [LIBDOVECOT_SERVICE_INCLUDE], [LIBDOVECOT_STORAGE_INCLUDE], [LIBDOVECOT_LOGIN_INCLUDE], [LIBDOVECOT_SQL_INCLUDE], [LIBDOVECOT_IMAP_LOGIN_INCLUDE], [LIBDOVECOT_CONFIG_INCLUDE], [LIBDOVECOT_IMAP_INCLUDE], [LIBDOVECOT_POP3_INCLUDE], [LIBDOVECOT_DSYNC_INCLUDE], [LIBDOVECOT_IMAPC_INCLUDE], [LIBDOVECOT_FTS_INCLUDE], [LIBDOVECOT_NOTIFY_INCLUDE], [LIBDOVECOT_ACL_INCLUDE], [LIBDOVECOT_LIBFTS_INCLUDE]) AM_CONDITIONAL(DOVECOT_INSTALLED, test "$DOVECOT_INSTALLED" = "yes") diff --git a/run-test-valgrind.supp b/run-test-valgrind.supp index 03585511d8c..0ccf1c67b4c 100644 --- a/run-test-valgrind.supp +++ b/run-test-valgrind.supp @@ -4,3 +4,47 @@ fun:malloc obj:*/bash } +{ + + Memcheck:Cond + obj:/lib/x86_64-linux-gnu/liblzma.so.5.0.0 + obj:/lib/x86_64-linux-gnu/liblzma.so.5.0.0 + obj:/lib/x86_64-linux-gnu/liblzma.so.5.0.0 + obj:/lib/x86_64-linux-gnu/liblzma.so.5.0.0 + obj:/lib/x86_64-linux-gnu/liblzma.so.5.0.0 + fun:lzma_stream_encoder + fun:lzma_easy_encoder +} +{ + + Memcheck:Leak + fun:malloc + obj:* + obj:* + obj:* + obj:* + obj:* + obj:* + obj:* + obj:* + obj:* + fun:module_dir_init + fun:dcrypt_initialize + fun:main +} +{ + + Memcheck:Leak + fun:malloc + obj:* + obj:* + obj:* + obj:* + obj:* + obj:* + obj:* + obj:* + fun:module_dir_init + fun:dcrypt_initialize + fun:main +} diff --git a/src/Makefile.am b/src/Makefile.am index cafcef86df1..dd76173d8a4 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -9,11 +9,11 @@ LIBDOVECOT_SUBDIRS = \ lib-auth \ lib-master \ lib-charset \ + lib-ssl-iostream \ + lib-dcrypt \ lib-dns \ - $(LIB_LDAP) \ lib-dict \ lib-sasl \ - lib-ssl-iostream \ lib-stats \ lib-http \ lib-fs \ @@ -23,7 +23,9 @@ LIBDOVECOT_SUBDIRS = \ SUBDIRS = \ $(LIBDOVECOT_SUBDIRS) \ + lib-dict-extra \ lib-dovecot \ + $(LIB_LDAP) \ lib-fts \ lib-imap-client \ lib-imap-urlauth \ diff --git a/src/auth/Makefile.am b/src/auth/Makefile.am index 3e00e246625..5c1987d6a3f 100644 --- a/src/auth/Makefile.am +++ b/src/auth/Makefile.am @@ -27,6 +27,7 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib-test \ -I$(top_srcdir)/src/lib-dict \ -I$(top_srcdir)/src/lib-dns \ + -I$(top_srcdir)/src/lib-http \ -I$(top_srcdir)/src/lib-sql \ -I$(top_srcdir)/src/lib-settings \ -I$(top_srcdir)/src/lib-stats \ @@ -47,9 +48,11 @@ libpassword_a_SOURCES = \ password-scheme-md5crypt.c \ password-scheme-scram.c \ password-scheme-otp.c \ - password-scheme-rpa.c + password-scheme-rpa.c \ + password-scheme-pbkdf2.c auth_libs = \ + libstats_auth.la \ libpassword.a \ ../lib-ntlm/libntlm.a \ ../lib-otp/libotp.a \ @@ -65,6 +68,7 @@ auth_SOURCES = \ auth-cache.c \ auth-client-connection.c \ auth-master-connection.c \ + auth-policy.c \ auth-postfix-connection.c \ mech-otp-skey-common.c \ mech-plain-common.c \ @@ -74,7 +78,6 @@ auth_SOURCES = \ auth-request-stats.c \ auth-request-var-expand.c \ auth-settings.c \ - auth-stats.c \ auth-fields.c \ auth-token.c \ auth-worker-client.c \ @@ -140,6 +143,7 @@ headers = \ mech-otp-skey-common.h \ mech-plain-common.h \ auth-penalty.h \ + auth-policy.h \ auth-request.h \ auth-request-handler.h \ auth-request-stats.h \ @@ -204,9 +208,9 @@ stats_moduledir = $(moduledir)/stats stats_module_LTLIBRARIES = libstats_auth.la libstats_auth_la_LDFLAGS = -module -avoid-version -libstats_auth_la_LIBADD = auth-stats.lo $(LIBDOVECOT) -libstats_auth_la_DEPENDENCIES = auth-stats.lo -libstats_auth_la_SOURCES = +libstats_auth_la_LIBADD = $(LIBDOVECOT) +libstats_auth_la_DEPENDENCIES = $(LIBDOVECOT_DEPS) +libstats_auth_la_SOURCES = auth-stats.c test_programs = \ test-auth-cache \ diff --git a/src/auth/auth-policy.c b/src/auth/auth-policy.c new file mode 100755 index 00000000000..e815de1b028 --- /dev/null +++ b/src/auth/auth-policy.c @@ -0,0 +1,538 @@ +#include "lib.h" +#include "net.h" +#include "str.h" +#include "istream.h" +#include "ioloop.h" +#include "base64.h" +#include "hex-binary.h" +#include "hash-method.h" +#include "http-url.h" +#include "http-client.h" +#include "json-parser.h" +#include "auth-request.h" +#include "auth-penalty.h" +#include "auth-settings.h" +#include "auth-policy.h" + +#define AUTH_POLICY_DNS_SOCKET_PATH "dns-client" + +static struct http_client_settings http_client_set = { + .dns_client_socket_path = AUTH_POLICY_DNS_SOCKET_PATH, + .max_connect_attempts = 1, + .max_idle_time_msecs = 10000, + .max_parallel_connections = 100, + .debug = 0, + .user_agent = "dovecot/auth-policy-client" +}; + +static char *auth_policy_json_template; + +static struct http_client *http_client; + +struct policy_lookup_ctx { + pool_t pool; + string_t *json; + struct auth_request *request; + struct http_client_request *http_request; + struct json_parser *parser; + const struct auth_settings *set; + const char *url; + bool expect_result; + int result; + const char *message; + auth_policy_callback_t callback; + void *callback_context; + + struct istream *payload; + struct io *io; + + enum { + POLICY_RESULT = 0, + POLICY_RESULT_VALUE_STATUS, + POLICY_RESULT_VALUE_MESSAGE + } parse_state; + + bool parse_error; +}; + +struct policy_template_keyvalue { + const char *key; + const char *value; +}; + +static +int auth_policy_attribute_comparator(const struct policy_template_keyvalue *a, + const struct policy_template_keyvalue *b) +{ + return strcmp(a->key, b->key); +} + +static +int auth_policy_strptrcmp(const char *a0, const char *a1, + const char *b0, const char *b1) +{ + i_assert(a0 <= a1 && b0 <= b1); + return memcmp(a0, b0, I_MIN((a1-a0),(b1-b0))); +} + +static +void auth_policy_open_key(const char *key, string_t *template) +{ + const char *ptr; + while((ptr = strchr(key, '/')) != NULL) { + str_append_c(template,'"'); + json_append_escaped(template, t_strndup(key, (ptr-key))); + str_append_c(template,'"'); + str_append_c(template,':'); + str_append_c(template,'{'); + key = ptr+1; + } +} + +static +void auth_policy_close_key(const char *key, string_t *template) +{ + while((key = strchr(key, '/')) != NULL) { str_append_c(template,'}'); key++; } +} + +static +void auth_policy_open_and_close_to_key(const char *fromkey, const char *tokey, string_t *template) +{ + const char *fptr,*tptr,*fdash,*tdash; + + fptr = strrchr(fromkey, '/'); + tptr = strrchr(tokey, '/'); + + if (fptr == NULL && tptr == NULL) return; /* nothing to do */ + + if (fptr == NULL && tptr != NULL) { + auth_policy_open_key(tokey, template); + return; + } + + if (fptr != NULL && tptr == NULL) { + str_truncate(template, str_len(template)-1); + + auth_policy_close_key(fromkey, template); + str_append_c(template, ','); + return; + } + + if (auth_policy_strptrcmp(fromkey, fptr, tokey, tptr) == 0) { + /* nothing to do, again */ + return; + } + + fptr = fromkey; + tptr = tokey; + + while (fptr != NULL && tptr != NULL) { + fdash = strchr(fptr, '/'); + tdash = strchr(tptr, '/'); + + if (fdash == NULL) { + auth_policy_open_key(tptr, template); + break; + } + if (tdash == NULL) { + str_truncate(template, str_len(template)-1); + auth_policy_close_key(fptr, template); + str_append_c(template, ','); + break; + } + if (auth_policy_strptrcmp(fptr, fdash, tptr, tdash) != 0) { + str_truncate(template, str_len(template)-1); + auth_policy_close_key(fptr, template); + str_append_c(template, ','); + auth_policy_open_key(tptr, template); + break; + } + fptr = fdash+1; + tptr = tdash+1; + } +} + +void auth_policy_init(void) +{ + http_client_set.request_absolute_timeout_msecs = global_auth_settings->policy_server_timeout_msecs; + if (global_auth_settings->debug) + http_client_set.debug = 1; + http_client = http_client_init(&http_client_set); + + /* prepare template */ + + ARRAY(struct policy_template_keyvalue) attribute_pairs; + const struct policy_template_keyvalue *kvptr; + string_t *template = t_str_new(64); + const char **ptr; + const char *key = NULL; + const char **list = t_strsplit_spaces(global_auth_settings->policy_request_attributes, "= "); + + t_array_init(&attribute_pairs, 8); + for(ptr = list; *ptr != NULL; ptr++) { + struct policy_template_keyvalue pair; + if (key == NULL) { + key = *ptr; + } else { + pair.key = key; + pair.value = *ptr; + key = NULL; + array_append(&attribute_pairs, &pair, 1); + } + } + if (key != NULL) { + i_fatal("auth_policy_request_attributes contains invalid value"); + } + + /* then we sort it */ + array_sort(&attribute_pairs, auth_policy_attribute_comparator); + + /* and build a template string */ + const char *prevkey = ""; + + array_foreach(&attribute_pairs, kvptr) { + const char *kptr = strchr(kvptr->key, '/'); + auth_policy_open_and_close_to_key(prevkey, kvptr->key, template); + str_append_c(template,'"'); + json_append_escaped(template, (kptr != NULL?kptr+1:kvptr->key)); + str_append_c(template,'"'); + str_append_c(template,':'); + str_append_c(template,'"'); + str_append(template,kvptr->value); + str_append_c(template,'"'); + str_append_c(template,','); + prevkey = kvptr->key; + } + + auth_policy_open_and_close_to_key(prevkey, "", template); + str_truncate(template, str_len(template)-1); + auth_policy_json_template = i_strdup(str_c(template)); +} + +void auth_policy_deinit(void) +{ + if (http_client != NULL) + http_client_deinit(&http_client); + i_free(auth_policy_json_template); +} + +static +void auth_policy_finish(void *ctx) +{ + struct policy_lookup_ctx *context = ctx; + + if (context->parser != NULL) { + const char *error ATTR_UNUSED; + json_parser_deinit(&(context->parser), &error); + } + if (context->http_request != NULL) + http_client_request_abort(&(context->http_request)); +} + +static +void auth_policy_parse_response(struct policy_lookup_ctx *context) +{ + enum json_type type; + const char *value; + int ret; + + while((ret = json_parse_next(context->parser, &type, &value)) == 1) { + if (context->parse_state == POLICY_RESULT) { + if (type != JSON_TYPE_OBJECT_KEY) + break; + else if (strcmp(value, "status") == 0) + context->parse_state = POLICY_RESULT_VALUE_STATUS; + else if (strcmp(value, "msg") == 0) + context->parse_state = POLICY_RESULT_VALUE_MESSAGE; + else break; + } else if (context->parse_state == POLICY_RESULT_VALUE_STATUS) { + if (type != JSON_TYPE_NUMBER || str_to_int(value, &(context->result)) != 0) + break; + context->parse_state = POLICY_RESULT; + } else if (context->parse_state == POLICY_RESULT_VALUE_MESSAGE) { + if (type != JSON_TYPE_STRING) + break; + if (*value != '\0') + context->message = p_strdup(context->pool, value); + context->parse_state = POLICY_RESULT; + } else { + break; + } + } + + if (ret == 0 && !context->payload->eof) + return; + + context->parse_error = TRUE; + + io_remove(&(context->io)); + + if (context->payload->stream_errno != 0) { + auth_request_log_error(context->request, "policy", + "Error reading policy server result: %s", + i_stream_get_error(context->payload)); + } else if (ret == 0 && context->payload->eof) { + auth_request_log_error(context->request, "policy", + "Policy server result was too short"); + } else if (ret == 1) { + auth_request_log_error(context->request, "policy", + "Policy server response was malformed"); + } else { + const char *error = "unknown"; + if (json_parser_deinit(&(context->parser), &error) != 0) + auth_request_log_error(context->request, "policy", + "Policy server response JSON parse error: %s", error); + else if (context->parse_state == POLICY_RESULT) + context->parse_error = FALSE; + } + i_stream_unref(&(context->payload)); + + if (context->parse_error) { + context->result = (context->set->policy_reject_on_fail ? -1 : 0); + } + + context->request->policy_refusal = FALSE; + + if (context->result < 0) { + if (context->message != NULL) { + /* set message here */ + auth_request_log_debug(context->request, "policy", + "Policy response %d with message: %s", + context->result, context->message); + auth_request_set_field(context->request, "reason", context->message, NULL); + } + context->request->policy_refusal = TRUE; + } else { + auth_request_log_debug(context->request, "policy", + "Policy response %d", context->result); + } + + if (context->request->policy_refusal == TRUE && context->set->verbose == TRUE) { + auth_request_log_info(context->request, "policy", "Authentication failure due to policy server refusal%s%s", + (context->message!=NULL?": ":""), + (context->message!=NULL?context->message:"")); + } + + if (context->callback != NULL) { + context->callback(context->result, context->callback_context); + } +}; + +static +void auth_policy_process_response(const struct http_response *response, + void *ctx) +{ + struct policy_lookup_ctx *context = ctx; + + context->payload = response->payload; + + if ((response->status / 10) != 20) { + auth_request_log_error(context->request, "policy", + "Policy server HTTP error: %d %s", response->status, response->reason); + if (context->callback != NULL) + context->callback(context->result, context->callback_context); + return; + } + + if (response->payload == NULL) { + if (context->expect_result) + auth_request_log_error(context->request, "policy", + "Policy server result was empty"); + if (context->callback != NULL) + context->callback(context->result, context->callback_context); + return; + } + + if (context->expect_result) { + i_stream_ref(response->payload); + context->io = io_add_istream(response->payload, auth_policy_parse_response, context); + context->parser = json_parser_init(response->payload); + auth_policy_parse_response(ctx); + } else { + auth_request_log_debug(context->request, "policy", + "Policy response %d", context->result); + if (context->callback != NULL) + context->callback(context->result, context->callback_context); + } +} + +static +void auth_policy_send_request(struct policy_lookup_ctx *context) +{ + const char *error; + struct http_url *url; + if (http_url_parse(context->url, NULL, HTTP_URL_ALLOW_USERINFO_PART, + context->pool, &url, &error) != 0) { + auth_request_log_error(context->request, "policy", + "Could not parse url %s: %s", context->url, error); + auth_policy_finish(context); + return; + } + context->http_request = http_client_request_url(http_client, + "POST", url, auth_policy_process_response, (void*)context); + http_client_request_set_destroy_callback(context->http_request, auth_policy_finish, context); + http_client_request_add_header(context->http_request, "Content-Type", "application/json"); + if (*context->set->policy_server_api_header != 0) { + const char *ptr; + if ((ptr = strstr(context->set->policy_server_api_header, ":")) != NULL) { + const char *header = t_strcut(context->set->policy_server_api_header, ':'); + http_client_request_add_header(context->http_request, header, ptr + 1); + } else { + http_client_request_add_header(context->http_request, + "X-API-Key", context->set->policy_server_api_header); + } + } + if (url->user != NULL) { + /* allow empty password */ + http_client_request_set_auth_simple(context->http_request, url->user, + (url->password != NULL ? url->password : "")); + } + struct istream *is = i_stream_create_from_buffer(context->json); + http_client_request_set_payload(context->http_request, is, FALSE); + i_stream_unref(&is); + http_client_request_submit(context->http_request); +} + +static +const char *auth_policy_escape_function(const char *string, + const struct auth_request *auth_request ATTR_UNUSED) +{ + string_t *tmp = t_str_new(64); + json_append_escaped(tmp, string); + return str_c(tmp); +} + +static +const struct var_expand_table *policy_get_var_expand_table(struct auth_request *auth_request, + const char *hashed_password) +{ + struct var_expand_table *table; + unsigned int count = 1; + + table = auth_request_get_var_expand_table_full(auth_request, auth_policy_escape_function, + &count); + table[0].key = '\0'; + table[0].long_key = "hashed_password"; + table[0].value = hashed_password; + if (table[0].value != NULL) + table[0].value = auth_policy_escape_function(table[0].value, auth_request); + + return table; +} + +static +void auth_policy_create_json(struct policy_lookup_ctx *context, + const char *password, bool include_success) +{ + const struct var_expand_table *var_table; + context->json = str_new(context->pool, 64); + unsigned char *ptr; + const struct hash_method *digest = hash_method_lookup(context->set->policy_hash_mech); + + i_assert(digest != NULL); + + void *ctx = t_malloc(digest->context_size); + string_t *buffer = t_str_new(64); + + digest->init(ctx); + digest->loop(ctx, + context->set->policy_hash_nonce, + strlen(context->set->policy_hash_nonce)); + /* use +1 to make sure \0 gets included */ + digest->loop(ctx, context->request->user, strlen(context->request->user) + 1); + if (password != NULL) + digest->loop(ctx, password, strlen(password)); + ptr = (unsigned char*)str_c_modifiable(buffer); + digest->result(ctx, ptr); + str_truncate(buffer, digest->digest_size); + if (context->set->policy_hash_truncate > 0) { + /* truncate it to closest byte boundary */ + int bytes = ((context->set->policy_hash_truncate + 7) & -8)/8; + /* remainding bits */ + int bits = bytes*8 - context->set->policy_hash_truncate; + str_truncate(buffer, bytes); + ptr = buffer_get_modifiable_data(buffer, NULL); + /* right shift everything and left-pad with 0 */ + if (bits > 0) { + for(size_t i=bytes-1;i>0;i--) + ptr[i] = (ptr[i]>>(8-bits)) + ((ptr[i-1]&(0xff>>(8-bits)))<>(8-bits); + } + } + const char *hashed_password = binary_to_hex(str_data(buffer), str_len(buffer)); + str_append_c(context->json, '{'); + var_table = policy_get_var_expand_table(context->request, hashed_password); + auth_request_var_expand_with_table(context->json, auth_policy_json_template, + context->request, var_table, + auth_policy_escape_function); + if (include_success) { + str_append(context->json, ",\"success\":"); + if (!context->request->failed && context->request->successful && + !context->request->internal_failure) + str_append(context->json, "true"); + else + str_append(context->json, "false"); + str_append(context->json, ",\"policy_reject\":"); + str_append(context->json, context->request->policy_refusal ? "true" : "false"); + } + str_append_c(context->json, '}'); + auth_request_log_debug(context->request, "policy", + "Policy server request JSON: %s", str_c(context->json)); +} + +static +void auth_policy_url(struct policy_lookup_ctx *context, const char *command) +{ + size_t len = strlen(context->set->policy_server_url); + if (context->set->policy_server_url[len-1] == '&') + context->url = p_strdup_printf(context->pool, "%scommand=%s", + context->set->policy_server_url, command); + else + context->url = p_strdup_printf(context->pool, "%s?command=%s", + context->set->policy_server_url, command); +} + +void auth_policy_check(struct auth_request *request, const char *password, + auth_policy_callback_t cb, void *context) +{ + if (request->master != NULL || *(request->set->policy_server_url) == '\0') { + cb(0, context); + return; + } + struct policy_lookup_ctx *ctx = p_new(request->pool, struct policy_lookup_ctx, 1); + ctx->pool = request->pool; + ctx->request = request; + ctx->expect_result = TRUE; + ctx->callback = cb; + ctx->callback_context = context; + ctx->set = request->set; + + auth_policy_url(ctx, "allow"); + ctx->result = (ctx->set->policy_reject_on_fail ? -1 : 0); + auth_request_log_debug(request, "policy", "Policy request %s", ctx->url); + T_BEGIN { + auth_policy_create_json(ctx, password, FALSE); + } T_END; + auth_policy_send_request(ctx); +} + +void auth_policy_report(struct auth_request *request) +{ + if (request->master != NULL) + return; + + if (*(request->set->policy_server_url) == '\0') + return; + pool_t pool = pool_alloconly_create("auth policy", 128); + struct policy_lookup_ctx *ctx = p_new(pool, struct policy_lookup_ctx, 1); + ctx->pool = pool; + ctx->request = request; + ctx->expect_result = FALSE; + ctx->set = request->set; + auth_policy_url(ctx, "report"); + auth_request_log_debug(request, "policy", "Policy request %s", ctx->url); + T_BEGIN { + auth_policy_create_json(ctx, request->mech_password, TRUE); + } T_END; + auth_policy_send_request(ctx); +} diff --git a/src/auth/auth-policy.h b/src/auth/auth-policy.h new file mode 100644 index 00000000000..1c81945d709 --- /dev/null +++ b/src/auth/auth-policy.h @@ -0,0 +1,11 @@ +#ifndef AUTH_POLICY_H +#define AUTH_POLICY_H + +typedef void (*auth_policy_callback_t)(int, void *); + +void auth_policy_check(struct auth_request *request, const char *password, auth_policy_callback_t cb, void *context); +void auth_policy_report(struct auth_request *request); +void auth_policy_init(void); +void auth_policy_deinit(void); + +#endif diff --git a/src/auth/auth-request-handler.c b/src/auth/auth-request-handler.c index b42f490ed0b..17d52c655e9 100644 --- a/src/auth/auth-request-handler.c +++ b/src/auth/auth-request-handler.c @@ -16,7 +16,7 @@ #include "auth-token.h" #include "auth-master-connection.h" #include "auth-request-handler.h" - +#include "auth-policy.h" #define AUTH_FAILURE_DELAY_CHECK_MSECS 500 @@ -215,6 +215,8 @@ auth_request_handle_failure(struct auth_request *request, const char *reply) auth_request_ref(request); auth_request_handler_remove(handler, request); + auth_policy_report(request); + if (auth_fields_exists(request->extra_fields, "nodelay")) { /* passdb specifically requested not to delay the reply. */ handler->callback(reply, handler->conn); @@ -260,6 +262,9 @@ auth_request_handler_reply_success_finish(struct auth_request *request) str_printfa(str, "OK\t%u\tuser=", request->id); str_append_tabescaped(str, request->user); auth_str_append_extra_fields(request, str); + + auth_policy_report(request); + if (handler->master_callback == NULL || auth_fields_exists(request->extra_fields, "nologin") || auth_fields_exists(request->extra_fields, "proxy")) { @@ -267,6 +272,7 @@ auth_request_handler_reply_success_finish(struct auth_request *request) process to pick it up. delete it */ auth_request_handler_remove(handler, request); } + handler->callback(str_c(str), handler->conn); } diff --git a/src/auth/auth-request-var-expand.c b/src/auth/auth-request-var-expand.c index 964bbf33b69..f1f041e473b 100644 --- a/src/auth/auth-request-var-expand.c +++ b/src/auth/auth-request-var-expand.c @@ -42,6 +42,9 @@ auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT+1] = { { '\0', NULL, "orig_user" }, { '\0', NULL, "orig_username" }, { '\0', NULL, "orig_domain" }, + { '\0', NULL, "auth_user" }, + { '\0', NULL, "auth_username" }, + { '\0', NULL, "auth_domain" }, /* be sure to update AUTH_REQUEST_VAR_TAB_COUNT */ { '\0', NULL, NULL } }; @@ -68,7 +71,7 @@ auth_request_get_var_expand_table_full(const struct auth_request *auth_request, const unsigned int auth_count = N_ELEMENTS(auth_request_var_expand_static_tab); struct var_expand_table *tab, *ret_tab; - const char *orig_user; + const char *orig_user, *auth_user; if (escape_func == NULL) escape_func = escape_none; @@ -154,6 +157,16 @@ auth_request_get_var_expand_table_full(const struct auth_request *auth_request, tab[29].value = strchr(orig_user, '@'); if (tab[29].value != NULL) tab[29].value = escape_func(tab[29].value+1, auth_request); + + if (auth_request->master_user != NULL) + auth_user = auth_request->master_user; + else + auth_user = orig_user; + tab[30].value = escape_func(auth_user, auth_request); + tab[31].value = escape_func(t_strcut(auth_user, '@'), auth_request); + tab[32].value = strchr(auth_user, '@'); + if (tab[32].value != NULL) + tab[32].value = escape_func(tab[32].value+1, auth_request); return ret_tab; } diff --git a/src/auth/auth-request-var-expand.h b/src/auth/auth-request-var-expand.h index 180454a4696..1362ec3f861 100644 --- a/src/auth/auth-request-var-expand.h +++ b/src/auth/auth-request-var-expand.h @@ -8,7 +8,7 @@ auth_request_escape_func_t(const char *string, #define AUTH_REQUEST_VAR_TAB_USER_IDX 0 #define AUTH_REQUEST_VAR_TAB_USERNAME_IDX 1 #define AUTH_REQUEST_VAR_TAB_DOMAIN_IDX 2 -#define AUTH_REQUEST_VAR_TAB_COUNT 30 +#define AUTH_REQUEST_VAR_TAB_COUNT 33 extern const struct var_expand_table auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT+1]; diff --git a/src/auth/auth-request.c b/src/auth/auth-request.c index 24242d9d131..89ec55d869b 100644 --- a/src/auth/auth-request.c +++ b/src/auth/auth-request.c @@ -18,6 +18,7 @@ #include "auth-request-stats.h" #include "auth-client-connection.h" #include "auth-master-connection.h" +#include "auth-policy.h" #include "passdb.h" #include "passdb-blocking.h" #include "passdb-cache.h" @@ -32,6 +33,7 @@ #define AUTH_DNS_SOCKET_PATH "dns-client" #define AUTH_DNS_DEFAULT_TIMEOUT_MSECS (1000*10) #define AUTH_DNS_WARN_MSECS 500 +#define AUTH_REQUEST_MAX_DELAY_SECS (60*5) #define CACHED_PASSWORD_SCHEME "SHA1" struct auth_request_proxy_dns_lookup_ctx { @@ -40,6 +42,20 @@ struct auth_request_proxy_dns_lookup_ctx { struct dns_lookup *dns_lookup; }; +struct auth_policy_check_ctx { + enum { + AUTH_POLICY_CHECK_TYPE_PLAIN, + AUTH_POLICY_CHECK_TYPE_LOOKUP, + AUTH_POLICY_CHECK_TYPE_SUCCESS, + } type; + struct auth_request *request; + + buffer_t *success_data; + + verify_plain_callback_t *callback_plain; + lookup_credentials_callback_t *callback_lookup; +}; + const char auth_default_subsystems[2]; unsigned int auth_request_state_count[AUTH_REQUEST_STATE_MAX]; @@ -49,6 +65,15 @@ static void get_log_prefix(string_t *str, struct auth_request *auth_request, static void auth_request_userdb_import(struct auth_request *request, const char *args); +static +void auth_request_verify_plain_continue(struct auth_request *request, + verify_plain_callback_t *callback); +static +void auth_request_lookup_credentials_policy_continue(struct auth_request *request, + lookup_credentials_callback_t *callback); +static +void auth_request_policy_check_callback(int result, void *context); + struct auth_request * auth_request_new(const struct mech_module *mech) { @@ -98,6 +123,8 @@ void auth_request_set_state(struct auth_request *request, if (request->state == state) return; + i_assert(request->to_penalty == NULL); + i_assert(auth_request_state_count[request->state] > 0); auth_request_state_count[request->state]--; auth_request_state_count[state]++; @@ -127,21 +154,47 @@ struct auth *auth_request_get_auth(struct auth_request *request) void auth_request_success(struct auth_request *request, const void *data, size_t data_size) { - struct auth_stats *stats; + i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE); + /* perform second policy lookup here */ + + struct auth_policy_check_ctx *ctx = p_new(request->pool, struct auth_policy_check_ctx, 1); + ctx->request = request; + ctx->success_data = buffer_create_dynamic(request->pool, data_size); + buffer_append(ctx->success_data, data, data_size); + ctx->type = AUTH_POLICY_CHECK_TYPE_SUCCESS; + auth_policy_check(request, request->mech_password, auth_request_policy_check_callback, ctx); +} + +static +void auth_request_success_continue(struct auth_policy_check_ctx *ctx) +{ + struct auth_request *request = ctx->request; + struct auth_stats *stats; i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE); + if (request->to_penalty != NULL) + timeout_remove(&request->to_penalty); + if (request->failed || !request->passdb_success) { /* password was valid, but some other check failed. */ auth_request_fail(request); return; } + if (request->delay_until > ioloop_time) { + unsigned int delay_secs = request->delay_until - ioloop_time; + request->to_penalty = timeout_add(delay_secs * 1000, + auth_request_success_continue, ctx); + return; + } + request->successful = TRUE; - if (data_size > 0 && !request->final_resp_ok) { + if (ctx->success_data->used > 0 && !request->final_resp_ok) { /* we'll need one more SASL round, since client doesn't support the final SASL response */ - auth_request_handler_reply_continue(request, data, data_size); + auth_request_handler_reply_continue(request, + ctx->success_data->data, ctx->success_data->used); return; } @@ -153,7 +206,7 @@ void auth_request_success(struct auth_request *request, auth_request_set_state(request, AUTH_REQUEST_STATE_FINISHED); auth_request_refresh_last_access(request); auth_request_handler_reply(request, AUTH_CLIENT_RESULT_SUCCESS, - data, data_size); + ctx->success_data->data, ctx->success_data->used); } void auth_request_fail(struct auth_request *request) @@ -230,7 +283,7 @@ void auth_request_export(struct auth_request *request, string_t *dest) auth_str_add_keyvalue(dest, "service", request->service); - if (request->master_user != NULL) { + if (request->master_user != NULL) { auth_str_add_keyvalue(dest, "master-user", request->master_user); } @@ -265,6 +318,8 @@ void auth_request_export(struct auth_request *request, string_t *dest) str_printfa(dest, "\treal_lport=%u", request->real_local_port); if (request->real_remote_port != 0) str_printfa(dest, "\treal_rport=%u", request->real_remote_port); + if (request->session_id != NULL) + str_printfa(dest, "\tsession=%s", request->session_id); if (request->debug) str_append(dest, "\tdebug"); if (request->secured) @@ -318,6 +373,7 @@ bool auth_request_import_info(struct auth_request *request, request->debug = TRUE; else return FALSE; + /* NOTE: keep in sync with auth_request_export() */ return TRUE; } @@ -461,9 +517,9 @@ static void auth_request_save_cache(struct auth_request *request, strdup() it so that mech_password doesn't get cleared too early. */ if (!password_generate_encoded(request->mech_password, - request->user, - CACHED_PASSWORD_SCHEME, - &encoded_password)) + request->user, + CACHED_PASSWORD_SCHEME, + &encoded_password)) i_unreached(); request->passdb_password = p_strconcat(request->pool, "{"CACHED_PASSWORD_SCHEME"}", @@ -587,7 +643,7 @@ auth_request_handle_passdb_callback(enum passdb_result *result, return TRUE; case PASSDB_RESULT_PASS_EXPIRED: auth_request_set_field(request, "reason", - "Password expired", NULL); + "Password expired", NULL); return TRUE; case PASSDB_RESULT_OK: @@ -647,7 +703,7 @@ auth_request_handle_passdb_callback(enum passdb_result *result, next_passdb = request->passdb->next; } while (next_passdb != NULL && - auth_request_want_skip_passdb(request, next_passdb)) + auth_request_want_skip_passdb(request, next_passdb)) next_passdb = next_passdb->next; if (*result == PASSDB_RESULT_OK) { @@ -669,7 +725,7 @@ auth_request_handle_passdb_callback(enum passdb_result *result, if (passdb_continue && next_passdb != NULL) { /* try next passdb. */ - request->passdb = next_passdb; + request->passdb = next_passdb; request->passdb_password = NULL; if (*result == PASSDB_RESULT_USER_UNKNOWN) { @@ -707,7 +763,7 @@ auth_request_verify_plain_callback_finish(enum passdb_result result, } else { auth_request_ref(request); request->passdb_result = result; - request->private_callback.verify_plain(result, request); + request->private_callback.verify_plain(request->passdb_result, request); auth_request_unref(&request); } } @@ -772,13 +828,82 @@ static bool auth_request_is_disabled_master_user(struct auth_request *request) return TRUE; } +static +void auth_request_policy_penalty_finish(void *context) +{ + struct auth_policy_check_ctx *ctx = context; + + if (ctx->request->to_penalty != NULL) + timeout_remove(&ctx->request->to_penalty); + + i_assert(ctx->request->state == AUTH_REQUEST_STATE_MECH_CONTINUE); + + switch(ctx->type) { + case AUTH_POLICY_CHECK_TYPE_PLAIN: + auth_request_verify_plain_continue(ctx->request, ctx->callback_plain); + return; + case AUTH_POLICY_CHECK_TYPE_LOOKUP: + auth_request_lookup_credentials_policy_continue(ctx->request, ctx->callback_lookup); + return; + case AUTH_POLICY_CHECK_TYPE_SUCCESS: + auth_request_success_continue(ctx); + return; + default: + i_unreached(); + } +} + +static +void auth_request_policy_check_callback(int result, void *context) +{ + struct auth_policy_check_ctx *ctx = context; + + ctx->request->policy_processed = TRUE; + + if (result == -1) { + /* fail it right here and now */ + auth_request_fail(ctx->request); + } else if (ctx->type != AUTH_POLICY_CHECK_TYPE_SUCCESS && result > 0 && !ctx->request->no_penalty) { + ctx->request->to_penalty = timeout_add(result * 1000, + auth_request_policy_penalty_finish, + context); + } else { + auth_request_policy_penalty_finish(context); + } +} + void auth_request_verify_plain(struct auth_request *request, - const char *password, - verify_plain_callback_t *callback) + const char *password, + verify_plain_callback_t *callback) { + struct auth_policy_check_ctx *ctx; + + i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE); + + if (request->mech_password == NULL) + request->mech_password = p_strdup(request->pool, password); + else + i_assert(request->mech_password == password); + + if (request->policy_processed) { + auth_request_verify_plain_continue(request, callback); + } else { + ctx = p_new(request->pool, struct auth_policy_check_ctx, 1); + ctx->request = request; + ctx->callback_plain = callback; + ctx->type = AUTH_POLICY_CHECK_TYPE_PLAIN; + auth_policy_check(request, request->mech_password, auth_request_policy_check_callback, ctx); + } +} + +static +void auth_request_verify_plain_continue(struct auth_request *request, + verify_plain_callback_t *callback) { + struct auth_passdb *passdb; enum passdb_result result; const char *cache_key; + const char *password = request->mech_password; i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE); @@ -794,11 +919,8 @@ void auth_request_verify_plain(struct auth_request *request, return; } - passdb = request->passdb; - if (request->mech_password == NULL) - request->mech_password = p_strdup(request->pool, password); - else - i_assert(request->mech_password == password); + passdb = request->passdb; + request->private_callback.verify_plain = callback; cache_key = passdb_cache == NULL ? NULL : passdb->cache_key; @@ -827,9 +949,9 @@ void auth_request_verify_plain(struct auth_request *request, static void auth_request_lookup_credentials_finish(enum passdb_result result, - const unsigned char *credentials, - size_t size, - struct auth_request *request) + const unsigned char *credentials, + size_t size, + struct auth_request *request) { passdb_template_export(request->passdb->override_fields_tmpl, request); if (!auth_request_handle_passdb_callback(&result, request)) { @@ -847,7 +969,7 @@ auth_request_lookup_credentials_finish(enum passdb_result result, } auth_request_lookup_credentials(request, request->credentials_scheme, - request->private_callback.lookup_credentials); + request->private_callback.lookup_credentials); } else { if (request->delayed_credentials != NULL && size == 0) { /* we did multiple passdb lookups, but the last one @@ -910,12 +1032,36 @@ void auth_request_lookup_credentials_callback(enum passdb_result result, } auth_request_lookup_credentials_finish(result, credentials, size, - request); + request); } void auth_request_lookup_credentials(struct auth_request *request, const char *scheme, lookup_credentials_callback_t *callback) +{ + struct auth_policy_check_ctx *ctx; + + i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE); + + if (request->credentials_scheme == NULL) + request->credentials_scheme = p_strdup(request->pool, scheme); + else + i_assert(request->credentials_scheme == scheme); + + if (request->policy_processed) + auth_request_lookup_credentials_policy_continue(request, callback); + else { + ctx = p_new(request->pool, struct auth_policy_check_ctx, 1); + ctx->request = request; + ctx->callback_lookup = callback; + ctx->type = AUTH_POLICY_CHECK_TYPE_LOOKUP; + auth_policy_check(request, ctx->request->mech_password, auth_request_policy_check_callback, ctx); + } +} + +static +void auth_request_lookup_credentials_policy_continue(struct auth_request *request, + lookup_credentials_callback_t *callback) { struct auth_passdb *passdb; const char *cache_key, *cache_cred, *cache_scheme; @@ -929,7 +1075,6 @@ void auth_request_lookup_credentials(struct auth_request *request, } passdb = request->passdb; - request->credentials_scheme = p_strdup(request->pool, scheme); request->private_callback.lookup_credentials = callback; cache_key = passdb_cache == NULL ? NULL : passdb->cache_key; @@ -1039,13 +1184,13 @@ static bool auth_request_lookup_user_cache(struct auth_request *request, if (value == NULL || (expired && !use_expired)) { stats->auth_cache_miss_count++; auth_request_log_debug(request, AUTH_SUBSYS_DB, - value == NULL ? "userdb cache miss" : - "userdb cache expired"); + value == NULL ? "userdb cache miss" : + "userdb cache expired"); return FALSE; } stats->auth_cache_hit_count++; auth_request_log_debug(request, AUTH_SUBSYS_DB, - "userdb cache hit: %s", value); + "userdb cache hit: %s", value); if (*value == '\0') { /* negative cache entry */ @@ -1112,7 +1257,7 @@ void auth_request_userdb_callback(enum userdb_result result, next_userdb = userdb->next; while (next_userdb != NULL && - auth_request_want_skip_userdb(request, next_userdb)) + auth_request_want_skip_userdb(request, next_userdb)) next_userdb = next_userdb->next; if (userdb_continue && next_userdb != NULL) { @@ -1137,10 +1282,11 @@ void auth_request_userdb_callback(enum userdb_result result, return; } - if (request->userdb_success) + if (request->userdb_success) { + result = USERDB_RESULT_OK; userdb_template_export(userdb->override_fields_tmpl, request); - else if (request->userdbs_seen_internal_failure || - result == USERDB_RESULT_INTERNAL_FAILURE) { + } else if (request->userdbs_seen_internal_failure || + result == USERDB_RESULT_INTERNAL_FAILURE) { /* one of the userdb lookups failed. the user might have been in there, so this is an internal failure */ result = USERDB_RESULT_INTERNAL_FAILURE; @@ -1154,6 +1300,9 @@ void auth_request_userdb_callback(enum userdb_result result, auth_request_log_error(request, AUTH_SUBSYS_MECH, "user not found from any userdbs"); } + result = USERDB_RESULT_USER_UNKNOWN; + } else { + result = USERDB_RESULT_USER_UNKNOWN; } if (request->userdb_lookup_tempfailed) { @@ -1173,7 +1322,7 @@ void auth_request_userdb_callback(enum userdb_result result, } } - request->private_callback.userdb(result, request); + request->private_callback.userdb(result, request); } void auth_request_lookup_user(struct auth_request *request, @@ -1217,7 +1366,7 @@ void auth_request_lookup_user(struct auth_request *request, static char * auth_request_fix_username(struct auth_request *request, const char *username, - const char **error_r) + const char **error_r) { const struct auth_settings *set = request->set; unsigned char *p; @@ -1226,12 +1375,12 @@ auth_request_fix_username(struct auth_request *request, const char *username, if (*set->default_realm != '\0' && strchr(username, '@') == NULL) { user = p_strconcat(request->pool, username, "@", - set->default_realm, NULL); + set->default_realm, NULL); } else { user = p_strdup(request->pool, username); } - for (p = (unsigned char *)user; *p != '\0'; p++) { + for (p = (unsigned char *)user; *p != '\0'; p++) { if (set->username_translation_map[*p & 0xff] != 0) *p = set->username_translation_map[*p & 0xff]; if (set->username_chars_map[*p & 0xff] == 0) { @@ -1265,11 +1414,11 @@ auth_request_fix_username(struct auth_request *request, const char *username, *error_r = "Empty username"; return NULL; } - return user; + return user; } bool auth_request_set_username(struct auth_request *request, - const char *username, const char **error_r) + const char *username, const char **error_r) { const struct auth_settings *set = request->set; const char *p, *login_username = NULL; @@ -1298,7 +1447,7 @@ bool auth_request_set_username(struct auth_request *request, username = request->user; } - request->user = auth_request_fix_username(request, username, error_r); + request->user = auth_request_fix_username(request, username, error_r); if (request->user == NULL) return FALSE; if (request->translated_username == NULL) { @@ -1316,8 +1465,8 @@ bool auth_request_set_username(struct auth_request *request, } bool auth_request_set_login_username(struct auth_request *request, - const char *username, - const char **error_r) + const char *username, + const char **error_r) { struct auth_passdb *master_passdb; @@ -1332,7 +1481,7 @@ bool auth_request_set_login_username(struct auth_request *request, return TRUE; } - /* lookup request->user from masterdb first */ + /* lookup request->user from masterdb first */ master_passdb = auth_request_get_auth(request)->masterdbs; if (master_passdb == NULL) { *error_r = "Master user login attempted without master passdbs"; @@ -1340,21 +1489,21 @@ bool auth_request_set_login_username(struct auth_request *request, } request->passdb = master_passdb; - request->requested_login_user = - auth_request_fix_username(request, username, error_r); + request->requested_login_user = + auth_request_fix_username(request, username, error_r); if (request->requested_login_user == NULL) return FALSE; auth_request_log_debug(request, AUTH_SUBSYS_DB, - "Master user lookup for login: %s", - request->requested_login_user); + "Master user lookup for login: %s", + request->requested_login_user); return TRUE; } static void auth_request_validate_networks(struct auth_request *request, - const char *name, const char *networks, - const struct ip_addr *remote_ip) + const char *name, const char *networks, + const struct ip_addr *remote_ip) { const char *const *net; struct ip_addr net_ip; @@ -1469,8 +1618,8 @@ auth_request_try_update_username(struct auth_request *request, if (strcmp(request->user, new_value) != 0) { auth_request_log_debug(request, AUTH_SUBSYS_DB, - "username changed %s -> %s", - request->user, new_value); + "username changed %s -> %s", + request->user, new_value); request->user = p_strdup(request->pool, new_value); } return TRUE; @@ -1504,6 +1653,11 @@ void auth_request_set_field(struct auth_request *request, name = t_strndup(name, name_len-10); if (auth_fields_exists(request->extra_fields, name)) return; + } else if (name_len > 7 && strcmp(name+name_len-7, ":remove") == 0) { + /* remove this field entirely */ + name = t_strndup(name, name_len-7); + auth_fields_remove(request->extra_fields, name); + return; } if (strcmp(name, "password") == 0) { @@ -1525,6 +1679,39 @@ void auth_request_set_field(struct auth_request *request, auth_request_validate_networks(request, name, value, &request->remote_ip); } else if (strcmp(name, "fail") == 0) { request->failed = TRUE; + } else if (strcmp(name, "delay_until") == 0) { + time_t timestamp; + unsigned int extra_secs = 0; + const char *p; + + p = strchr(value, '+'); + if (p != NULL) { + value = t_strdup_until(value, p++); + if (str_to_uint(p, &extra_secs) < 0) { + auth_request_log_error(request, AUTH_SUBSYS_DB, + "Invalid delay_until randomness number '%s'", p); + request->failed = TRUE; + } else { + extra_secs = rand() % extra_secs; + } + } + if (str_to_time(value, ×tamp) < 0) { + auth_request_log_error(request, AUTH_SUBSYS_DB, + "Invalid delay_until timestamp '%s'", value); + request->failed = TRUE; + } else if (timestamp <= ioloop_time) { + /* no more delays */ + } else if (timestamp - ioloop_time > AUTH_REQUEST_MAX_DELAY_SECS) { + auth_request_log_error(request, AUTH_SUBSYS_DB, + "delay_until timestamp %s is too much in the future, failing", value); + request->failed = TRUE; + } else { + /* add randomness, but not too much of it */ + timestamp += extra_secs; + if (timestamp - ioloop_time > AUTH_REQUEST_MAX_DELAY_SECS) + timestamp = ioloop_time + AUTH_REQUEST_MAX_DELAY_SECS; + request->delay_until = timestamp; + } } else if (strcmp(name, "allow_real_nets") == 0) { auth_request_validate_networks(request, name, value, &request->real_remote_ip); } else if (strncmp(name, "userdb_", 7) == 0) { @@ -1627,7 +1814,7 @@ static void auth_request_set_uidgid_file(struct auth_request *request, auth_request_var_expand(path, path_template, request, NULL); if (stat(str_c(path), &st) < 0) { auth_request_log_error(request, AUTH_SUBSYS_DB, - "stat(%s) failed: %m", str_c(path)); + "stat(%s) failed: %m", str_c(path)); } else { auth_fields_add(request->userdb_reply, "uid", dec2str(st.st_uid), 0); @@ -1668,6 +1855,11 @@ void auth_request_set_userdb_field(struct auth_request *request, name = t_strndup(name, name_len-10); if (auth_fields_exists(request->userdb_reply, name)) return; + } else if (name_len > 7 && strcmp(name+name_len-7, ":remove") == 0) { + /* remove this field entirely */ + name = t_strndup(name, name_len-7); + auth_fields_remove(request->userdb_reply, name); + return; } if (strcmp(name, "uid") == 0) { @@ -1795,7 +1987,7 @@ auth_request_proxy_finish_ip(struct auth_request *request, /* proxying to ourself - log in without proxying by dropping all the proxying fields. */ bool proxy_always = auth_fields_exists(request->extra_fields, - "proxy_always"); + "proxy_always"); auth_request_proxy_finish_failure(request); if (proxy_always) { @@ -1882,7 +2074,7 @@ static int auth_request_proxy_host_lookup(struct auth_request *request, request->dns_lookup_ctx = ctx; if (dns_lookup(host, &dns_set, auth_request_proxy_dns_callback, ctx, - &ctx->dns_lookup) < 0) { + &ctx->dns_lookup) < 0) { /* failed early */ return -1; } @@ -2064,7 +2256,7 @@ int auth_request_password_verify(struct auth_request *request, if (auth_fields_exists(request->extra_fields, "nopassword")) { auth_request_log_debug(request, subsystem, - "Allowing any password"); + "Allowing any password"); return 1; } @@ -2077,7 +2269,7 @@ int auth_request_password_verify(struct auth_request *request, scheme, error); } else { auth_request_log_error(request, subsystem, - "Unknown scheme %s", scheme); + "Unknown scheme %s", scheme); } return -1; } @@ -2091,8 +2283,8 @@ int auth_request_password_verify(struct auth_request *request, const char *password_str = request->set->debug_passwords ? t_strdup_printf(" '%s'", crypted_password) : ""; auth_request_log_error(request, subsystem, - "Invalid password%s in passdb: %s", - password_str, error); + "Invalid password%s in passdb: %s", + password_str, error); } else if (ret == 0) { auth_request_log_password_mismatch(request, subsystem); } @@ -2185,8 +2377,25 @@ void auth_request_log_info(struct auth_request *auth_request, { va_list va; - if (!auth_request->set->verbose) - return; + if (auth_request->set->debug) { + /* auth_debug=yes overrides auth_verbose settings */ + } else { + const char *db_auth_verbose = auth_request->userdb_lookup ? + auth_request->userdb->set->auth_verbose : + auth_request->passdb->set->auth_verbose; + switch (db_auth_verbose[0]) { + case 'y': + break; + case 'n': + return; + case 'd': + if (!auth_request->set->verbose) + return; + break; + default: + i_unreached(); + } + } va_start(va, format); T_BEGIN { diff --git a/src/auth/auth-request.h b/src/auth/auth-request.h index d2778ef4ef8..de525776d00 100644 --- a/src/auth/auth-request.h +++ b/src/auth/auth-request.h @@ -71,6 +71,7 @@ struct auth_request { unsigned int client_pid; unsigned int id; time_t last_access; + time_t delay_until; pid_t session_pid; const char *service, *mech_name, *session_id; @@ -78,6 +79,7 @@ struct auth_request { in_port_t local_port, remote_port, real_local_port, real_remote_port; struct timeout *to_abort, *to_penalty; + unsigned int policy_penalty; unsigned int last_penalty; unsigned int initial_response_len; const unsigned char *initial_response; @@ -143,6 +145,8 @@ struct auth_request { will work. */ unsigned int userdb_prefetch_set:1; unsigned int stats_sent:1; + unsigned int policy_refusal:1; + unsigned int policy_processed:1; /* ... mechanism specific data ... */ }; diff --git a/src/auth/auth-settings.c b/src/auth/auth-settings.c index 7eed12f5dbe..c942819c1fb 100644 --- a/src/auth/auth-settings.c +++ b/src/auth/auth-settings.c @@ -2,6 +2,7 @@ #include "lib.h" #include "array.h" +#include "hash-method.h" #include "settings-parser.h" #include "master-service-private.h" #include "master-service-settings.h" @@ -121,6 +122,7 @@ static const struct setting_define auth_passdb_setting_defines[] = { DEF(SET_BOOL, deny), DEF(SET_BOOL, pass), DEF(SET_BOOL, master), + DEF(SET_ENUM, auth_verbose), SETTING_DEFINE_LIST_END }; @@ -139,7 +141,8 @@ static const struct auth_passdb_settings auth_passdb_default_settings = { .deny = FALSE, .pass = FALSE, - .master = FALSE + .master = FALSE, + .auth_verbose = "default:yes:no" }; const struct setting_parser_info auth_passdb_setting_parser_info = { @@ -171,6 +174,8 @@ static const struct setting_define auth_userdb_setting_defines[] = { DEF(SET_ENUM, result_failure), DEF(SET_ENUM, result_internalfail), + DEF(SET_ENUM, auth_verbose), + SETTING_DEFINE_LIST_END }; @@ -185,7 +190,9 @@ static const struct auth_userdb_settings auth_userdb_default_settings = { .skip = "never:found:notfound", .result_success = "return-ok:return:return-fail:continue:continue-ok:continue-fail", .result_failure = "continue:return:return-ok:return-fail:continue-ok:continue-fail", - .result_internalfail = "continue:return:return-ok:return-fail:continue-ok:continue-fail" + .result_internalfail = "continue:return:return-ok:return-fail:continue-ok:continue-fail", + + .auth_verbose = "default:yes:no" }; const struct setting_parser_info auth_userdb_setting_parser_info = { @@ -230,6 +237,15 @@ static const struct setting_define auth_setting_defines[] = { DEF(SET_STR, proxy_self), DEF(SET_TIME, failure_delay), + DEF(SET_STR, policy_server_url), + DEF(SET_STR, policy_server_api_header), + DEF(SET_UINT, policy_server_timeout_msecs), + DEF(SET_STR, policy_hash_mech), + DEF(SET_STR, policy_hash_nonce), + DEF(SET_STR, policy_request_attributes), + DEF(SET_BOOL, policy_reject_on_fail), + DEF(SET_UINT, policy_hash_truncate), + DEF(SET_BOOL, stats), DEF(SET_BOOL, verbose), DEF(SET_BOOL, debug), @@ -270,6 +286,15 @@ static const struct auth_settings auth_default_settings = { .proxy_self = "", .failure_delay = 2, + .policy_server_url = "", + .policy_server_api_header = "", + .policy_server_timeout_msecs = 2000, + .policy_hash_mech = "sha256", + .policy_hash_nonce = "", + .policy_request_attributes = "login=%{orig_username} pwhash=%{hashed_password} remote=%{real_rip}", + .policy_reject_on_fail = FALSE, + .policy_hash_truncate = 12, + .stats = FALSE, .verbose = FALSE, .debug = FALSE, @@ -412,6 +437,25 @@ static bool auth_settings_check(void *_set, pool_t pool, set->realms_arr = (const char *const *)p_strsplit_spaces(pool, set->realms, " "); + if (*set->policy_server_url != '\0') { + if (*set->policy_hash_nonce == '\0') { + + *error_r = "auth_policy_hash_nonce must be set when policy server is used"; + return FALSE; + } + const struct hash_method *digest = hash_method_lookup(set->policy_hash_mech); + if (digest == NULL) { + *error_r = "invalid auth_policy_hash_mech given"; + return FALSE; + } + if (set->policy_hash_truncate > 0 && set->policy_hash_truncate >= digest->digest_size*8) { + *error_r = t_strdup_printf("policy_hash_truncate is not smaller than digest size (%u >= %u)", + set->policy_hash_truncate, + digest->digest_size*8); + return FALSE; + } + } + if (!auth_settings_set_self_ips(set, pool, error_r)) return FALSE; return TRUE; diff --git a/src/auth/auth-settings.h b/src/auth/auth-settings.h index c39f05151bb..1313576a978 100644 --- a/src/auth/auth-settings.h +++ b/src/auth/auth-settings.h @@ -18,6 +18,7 @@ struct auth_passdb_settings { bool deny; bool pass; /* deprecated, use result_success=continue instead */ bool master; + const char *auth_verbose; }; struct auth_userdb_settings { @@ -31,6 +32,7 @@ struct auth_userdb_settings { const char *result_success; const char *result_failure; const char *result_internalfail; + const char *auth_verbose; }; struct auth_settings { @@ -51,6 +53,15 @@ struct auth_settings { const char *proxy_self; unsigned int failure_delay; + const char *policy_server_url; + const char *policy_server_api_header; + unsigned int policy_server_timeout_msecs; + const char *policy_hash_mech; + const char *policy_hash_nonce; + const char *policy_request_attributes; + bool policy_reject_on_fail; + unsigned int policy_hash_truncate; + bool stats; bool verbose, debug, debug_passwords; const char *verbose_passwords; diff --git a/src/auth/db-dict.c b/src/auth/db-dict.c index 50f9f306d49..6720de150d8 100644 --- a/src/auth/db-dict.c +++ b/src/auth/db-dict.c @@ -193,8 +193,9 @@ static bool parse_section(const char *type, const char *name, ctx->cur_key->parsed_format = DB_DICT_VALUE_FORMAT_JSON; } else { - return t_strconcat("Unknown key format: ", - ctx->cur_key->format, NULL); + *errormsg = t_strconcat("Unknown key format: ", + ctx->cur_key->format, NULL); + return FALSE; } } ctx->cur_key = NULL; diff --git a/src/auth/db-ldap.c b/src/auth/db-ldap.c index 3a8c3f51092..f5f974d9c94 100644 --- a/src/auth/db-ldap.c +++ b/src/auth/db-ldap.c @@ -1552,6 +1552,7 @@ db_ldap_result_iterate_init_full(struct ldap_connection *conn, ctx->skip_null_values = skip_null_values; ctx->iter_dn_values = iter_dn_values; hash_table_create(&ctx->ldap_attrs, pool, 0, strcase_hash, strcasecmp); + ctx->var = str_new(ctx->pool, 256); if (ctx->auth_request->debug) ctx->debug = t_str_new(256); @@ -1630,16 +1631,17 @@ static const char *db_ldap_field_ptr_expand(const char *data, void *context) return db_ldap_field_expand(field_name, ctx); } +static struct var_expand_func_table ldap_var_funcs_table[] = { + { "ldap", db_ldap_field_expand }, + { "ldap_ptr", db_ldap_field_ptr_expand }, + { NULL, NULL } +}; + static const char *const * db_ldap_result_return_value(struct db_ldap_result_iterate_context *ctx, const struct ldap_field *field, struct db_ldap_value *ldap_value) { - static struct var_expand_func_table var_funcs_table[] = { - { "ldap", db_ldap_field_expand }, - { "ldap_ptr", db_ldap_field_ptr_expand }, - { NULL, NULL } - }; const struct var_expand_table *var_table; const char *const *values; @@ -1673,12 +1675,8 @@ db_ldap_result_return_value(struct db_ldap_result_iterate_context *ctx, (and less importantly the same for other variables) */ var_table = db_ldap_value_get_var_expand_table(ctx->auth_request, values[0]); - if (ctx->var == NULL) - ctx->var = str_new(ctx->pool, 256); - else - str_truncate(ctx->var, 0); var_expand_with_funcs(ctx->var, field->value, var_table, - var_funcs_table, ctx); + ldap_var_funcs_table, ctx); ctx->val_1_arr[0] = str_c(ctx->var); values = ctx->val_1_arr; } @@ -1691,6 +1689,7 @@ bool db_ldap_result_iterate_next(struct db_ldap_result_iterate_context *ctx, { const struct ldap_field *field; struct db_ldap_value *ldap_value; + unsigned int pos; do { if (ctx->attr_idx == array_count(ctx->attr_map)) @@ -1706,9 +1705,23 @@ bool db_ldap_result_iterate_next(struct db_ldap_result_iterate_context *ctx, else if (ctx->debug && *field->ldap_attr_name != '\0') str_printfa(ctx->debug, "; %s missing", field->ldap_attr_name); - *name_r = field->name; + str_truncate(ctx->var, 0); *values_r = db_ldap_result_return_value(ctx, field, ldap_value); + if (strchr(field->name, '%') == NULL) + *name_r = field->name; + else { + /* expand %variables also for LDAP name fields. we'll use the + same ctx->var, which may already contain the value. */ + str_append_c(ctx->var, '\0'); + pos = str_len(ctx->var); + + var_expand_with_funcs(ctx->var, field->name, + auth_request_get_var_expand_table(ctx->auth_request, NULL), + ldap_var_funcs_table, ctx); + *name_r = str_c(ctx->var) + pos; + } + if (ctx->skip_null_values && (*values_r)[0] == NULL) { /* no values. don't confuse the caller with this reply. */ return db_ldap_result_iterate_next(ctx, name_r, values_r); diff --git a/src/auth/main.c b/src/auth/main.c index 5a87c575ced..acfc535a8b2 100644 --- a/src/auth/main.c +++ b/src/auth/main.c @@ -30,6 +30,7 @@ #include "auth-master-connection.h" #include "auth-client-connection.h" #include "auth-postfix-connection.h" +#include "auth-policy.h" #include #include @@ -246,6 +247,7 @@ static void main_init(void) auth_worker_server_init(); auths_init(); auth_request_handler_init(); + auth_policy_init(); if (worker) { /* workers have only a single connection from the master @@ -265,6 +267,7 @@ static void main_deinit(void) /* cancel all pending anvil penalty lookups */ auth_penalty_deinit(&auth_penalty); } + auth_policy_deinit(); /* deinit auth workers, which aborts pending requests */ auth_worker_server_deinit(); /* deinit passdbs and userdbs. it aborts any pending async requests. */ diff --git a/src/auth/mech-rpa.c b/src/auth/mech-rpa.c index 743b323f067..98b6f9e4406 100644 --- a/src/auth/mech-rpa.c +++ b/src/auth/mech-rpa.c @@ -62,7 +62,7 @@ void *ucs2be_str(pool_t pool, const char *str, size_t *size); * Compute client -> server authentication response. */ static void rpa_user_response(struct rpa_auth_request *request, - unsigned char digest[MD5_RESULTLEN]) + unsigned char digest[STATIC_ARRAY MD5_RESULTLEN]) { struct md5_context ctx; unsigned char z[48]; @@ -86,7 +86,7 @@ static void rpa_user_response(struct rpa_auth_request *request, * Compute server -> client authentication response. */ static void rpa_server_response(struct rpa_auth_request *request, - unsigned char digest[MD5_RESULTLEN]) + unsigned char digest[STATIC_ARRAY MD5_RESULTLEN]) { struct md5_context ctx; unsigned char tmp[MD5_RESULTLEN]; diff --git a/src/auth/mech-winbind.c b/src/auth/mech-winbind.c index c9e1450f0ba..4a656961051 100644 --- a/src/auth/mech-winbind.c +++ b/src/auth/mech-winbind.c @@ -180,7 +180,8 @@ do_auth_continue(struct auth_request *auth_request, str_data(str), str_len(str)) < 0 || o_stream_flush(request->winbind->out_pipe) < 0) { auth_request_log_error(auth_request, AUTH_SUBSYS_MECH, - "write(out_pipe) failed: %m"); + "write(out_pipe) failed: %s", + o_stream_get_error(request->winbind->out_pipe)); return HR_RESTART; } request->continued = FALSE; diff --git a/src/auth/passdb.c b/src/auth/passdb.c index ec47e2c3473..a9068d72e1f 100644 --- a/src/auth/passdb.c +++ b/src/auth/passdb.c @@ -252,7 +252,7 @@ void passdb_deinit(struct passdb_module *passdb) passdb->iface = passdb_iface_deinit; } -void passdbs_generate_md5(unsigned char md5[MD5_RESULTLEN]) +void passdbs_generate_md5(unsigned char md5[STATIC_ARRAY MD5_RESULTLEN]) { struct md5_context ctx; struct passdb_module *const *passdbs; diff --git a/src/auth/passdb.h b/src/auth/passdb.h index 60f05b3b784..57b25391e1e 100644 --- a/src/auth/passdb.h +++ b/src/auth/passdb.h @@ -102,7 +102,7 @@ void passdb_deinit(struct passdb_module *passdb); void passdb_register_module(struct passdb_module_interface *iface); void passdb_unregister_module(struct passdb_module_interface *iface); -void passdbs_generate_md5(unsigned char md5[MD5_RESULTLEN]); +void passdbs_generate_md5(unsigned char md5[STATIC_ARRAY MD5_RESULTLEN]); void passdbs_init(void); void passdbs_deinit(void); diff --git a/src/auth/password-scheme-pbkdf2.c b/src/auth/password-scheme-pbkdf2.c new file mode 100644 index 00000000000..36305881923 --- /dev/null +++ b/src/auth/password-scheme-pbkdf2.c @@ -0,0 +1,82 @@ +/* Copyright (c) 2015 Dovecot Oy, see the included COPYING file */ + +#include "lib.h" +#include "buffer.h" +#include "str.h" +#include "password-scheme.h" +#include "hex-binary.h" +#include "hash-method.h" +#include "pkcs5.h" + +#define PBKDF2_KEY_SIZE_SHA1 20 + +#define PBKDF2_GENERATE_SALT_LEN 16 +#define PBKDF2_ROUNDS_DEFAULT 5000 + +static void +pbkdf_run(const char *plaintext, const char *salt, + unsigned int rounds, unsigned char key_r[PBKDF2_KEY_SIZE_SHA1]) +{ + memset(key_r, 0, PBKDF2_KEY_SIZE_SHA1); + buffer_t buf; + buffer_create_from_data(&buf, key_r, PBKDF2_KEY_SIZE_SHA1); + + pkcs5_pbkdf(PKCS5_PBKDF2, hash_method_lookup("sha1"), + (const unsigned char *)plaintext, strlen(plaintext), + (const unsigned char *)salt, strlen(salt), + rounds, PBKDF2_KEY_SIZE_SHA1, &buf); +} + +void pbkdf2_generate(const char *plaintext, const char *user ATTR_UNUSED, + const unsigned char **raw_password_r, size_t *size_r) +{ + unsigned char key[PBKDF2_KEY_SIZE_SHA1]; + const char *salt; + string_t *str = t_str_new(64); + unsigned int rounds = password_scheme_encryption_rounds; + + if (rounds == 0) + rounds = PBKDF2_ROUNDS_DEFAULT; + salt = password_generate_salt(PBKDF2_GENERATE_SALT_LEN); + pbkdf_run(plaintext, salt, rounds, key); + + str_printfa(str, "$1$%s$%u$", salt, rounds); + binary_to_hex_append(str, key, sizeof(key)); + + *raw_password_r = str_data(str); + *size_r = str_len(str); +} + +int pbkdf2_verify(const char *plaintext, const char *user ATTR_UNUSED, + const unsigned char *raw_password, size_t size, + const char **error_r) +{ + const char *const *fields; + const char *salt; + unsigned int rounds; + unsigned char key1[PBKDF2_KEY_SIZE_SHA1], key2[PBKDF2_KEY_SIZE_SHA1]; + buffer_t buf; + + /* $1$salt$rounds$hash */ + if (size < 3 || memcmp(raw_password, "$1$", 3) != 0) { + *error_r = "Invalid PBKDF2 passdb entry prefix"; + return -1; + } + + fields = t_strsplit(t_strndup(raw_password + 3, size - 3), "$"); + salt = fields[0]; + if (str_array_length(fields) != 3 || + str_to_uint(fields[1], &rounds) < 0) { + *error_r = "Invalid PBKDF2 passdb entry format"; + return -1; + } + buffer_create_from_data(&buf, key1, sizeof(key1)); + if (strlen(fields[2]) != sizeof(key1)*2 || + hex_to_binary(fields[2], &buf) < 0) { + *error_r = "PBKDF2 hash not 160bit hex-encoded"; + return -1; + } + + pbkdf_run(plaintext, salt, rounds, key2); + return memcmp(key1, key2, sizeof(key1)) == 0 ? 1 : 0; +} diff --git a/src/auth/password-scheme.c b/src/auth/password-scheme.c index baefb9532cb..1633e50b1ea 100644 --- a/src/auth/password-scheme.c +++ b/src/auth/password-scheme.c @@ -825,7 +825,8 @@ static const struct password_scheme builtin_schemes[] = { { "NTLM", PW_ENCODING_HEX, NTLMSSP_HASH_SIZE, NULL, ntlm_generate }, { "OTP", PW_ENCODING_NONE, 0, otp_verify, otp_generate }, { "SKEY", PW_ENCODING_NONE, 0, otp_verify, skey_generate }, - { "RPA", PW_ENCODING_HEX, MD5_RESULTLEN, NULL, rpa_generate } + { "RPA", PW_ENCODING_HEX, MD5_RESULTLEN, NULL, rpa_generate }, + { "PBKDF2", PW_ENCODING_NONE, 0, pbkdf2_verify, pbkdf2_generate }, }; void password_scheme_register(const struct password_scheme *scheme) diff --git a/src/auth/password-scheme.h b/src/auth/password-scheme.h index 438e849919d..65a92ef1c5e 100644 --- a/src/auth/password-scheme.h +++ b/src/auth/password-scheme.h @@ -95,6 +95,11 @@ int scram_sha1_verify(const char *plaintext, const char *user ATTR_UNUSED, const char **error_r ATTR_UNUSED); void scram_sha1_generate(const char *plaintext, const char *user ATTR_UNUSED, const unsigned char **raw_password_r, size_t *size_r); +void pbkdf2_generate(const char *plaintext, const char *user ATTR_UNUSED, + const unsigned char **raw_password_r, size_t *size_r); +int pbkdf2_verify(const char *plaintext, const char *user ATTR_UNUSED, + const unsigned char *raw_password, size_t size, + const char **error_r); /* check wich of the algorithms Blowfisch, SHA-256 and SHA-512 are supported by the used libc's/glibc's crypt() */ diff --git a/src/auth/userdb-passwd-file.c b/src/auth/userdb-passwd-file.c index b578d272b6c..f80cac1f9e0 100644 --- a/src/auth/userdb-passwd-file.c +++ b/src/auth/userdb-passwd-file.c @@ -139,7 +139,8 @@ static void passwd_file_iterate_next(struct userdb_iterate_context *_ctx) break; } if (line == NULL && ctx->input->stream_errno != 0) { - i_error("read(%s) failed: %m", ctx->path); + i_error("read(%s) failed: %s", ctx->path, + i_stream_get_error(ctx->input)); _ctx->failed = TRUE; } } diff --git a/src/auth/userdb.c b/src/auth/userdb.c index 5b45ddba728..29e9b2e0668 100644 --- a/src/auth/userdb.c +++ b/src/auth/userdb.c @@ -194,7 +194,7 @@ void userdb_deinit(struct userdb_module *userdb) userdb->iface = &userdb_iface_deinit; } -void userdbs_generate_md5(unsigned char md5[MD5_RESULTLEN]) +void userdbs_generate_md5(unsigned char md5[STATIC_ARRAY MD5_RESULTLEN]) { struct md5_context ctx; struct userdb_module *const *userdbs; diff --git a/src/auth/userdb.h b/src/auth/userdb.h index 45ba70b3adb..ac527011e87 100644 --- a/src/auth/userdb.h +++ b/src/auth/userdb.h @@ -81,7 +81,7 @@ void userdb_deinit(struct userdb_module *userdb); void userdb_register_module(struct userdb_module_interface *iface); void userdb_unregister_module(struct userdb_module_interface *iface); -void userdbs_generate_md5(unsigned char md5[MD5_RESULTLEN]); +void userdbs_generate_md5(unsigned char md5[STATIC_ARRAY MD5_RESULTLEN]); void userdbs_init(void); void userdbs_deinit(void); diff --git a/src/config/config-parser.c b/src/config/config-parser.c index 169a3425cb6..c6e04256c48 100644 --- a/src/config/config-parser.c +++ b/src/config/config-parser.c @@ -595,17 +595,25 @@ config_parse_line(struct config_parser_context *ctx, /* remove whitespace from end of line */ len = strlen(line); - while (IS_WHITE(line[len-1])) + while (len >= 1) { + if(!IS_WHITE(line[len-1])) + break; len--; + } line[len] = '\0'; - if (len > 0 && line[len-1] == '\\') { + if (len >= 1 && line[len-1] == '\\') { /* continues in next line */ len--; - while (IS_WHITE(line[len-1])) + while (len >= 1) { + if(!IS_WHITE(line[len-1])) + break; len--; - str_append_n(full_line, line, len); - str_append_c(full_line, ' '); + } + if(len >= 1) { + str_append_n(full_line, line, len); + str_append_c(full_line, ' '); + } return CONFIG_LINE_TYPE_CONTINUE; } if (str_len(full_line) > 0) { @@ -849,9 +857,10 @@ config_parser_check_warnings(struct config_parser_context *ctx, const char *key) filters, log a warning. */ if (first_pos == NULL) return; - i_warning("%s line %u: Global setting %s won't change the setting inside an earlier filter at %s", + i_warning("%s line %u: Global setting %s won't change the setting inside an earlier filter at %s " + "(if this is intentional, avoid this warning by moving the global setting before %s)", ctx->cur_input->path, ctx->cur_input->linenum, - key, first_pos); + key, first_pos, first_pos); return; } if (first_pos != NULL) diff --git a/src/config/settings-get.pl b/src/config/settings-get.pl index 7c6edfeea0b..6bfb1598d1e 100755 --- a/src/config/settings-get.pl +++ b/src/config/settings-get.pl @@ -10,6 +10,7 @@ print '#include "hash-format.h"'."\n"; print '#include "net.h"'."\n"; print '#include "unichar.h"'."\n"; +print '#include "hash-method.h"'."\n"; print '#include "settings-parser.h"'."\n"; print '#include "all-settings.h"'."\n"; print '#include '."\n"; diff --git a/src/dict/Makefile.am b/src/dict/Makefile.am index 93e7a42363a..4606de89020 100644 --- a/src/dict/Makefile.am +++ b/src/dict/Makefile.am @@ -33,4 +33,5 @@ dict_SOURCES = \ noinst_HEADERS = \ dict-connection.h \ dict-commands.h \ - dict-settings.h + dict-settings.h \ + main.h diff --git a/src/dict/dict-commands.c b/src/dict/dict-commands.c index 9742a5ead74..7c19f468eae 100644 --- a/src/dict/dict-commands.c +++ b/src/dict/dict-commands.c @@ -5,11 +5,13 @@ #include "ostream.h" #include "str.h" #include "strescape.h" +#include "timing.h" +#include "time-util.h" #include "dict-client.h" #include "dict-settings.h" #include "dict-connection.h" #include "dict-commands.h" - +#include "main.h" #define DICT_OUTPUT_OPTIMAL_SIZE 1024 @@ -21,6 +23,7 @@ struct dict_cmd_func { struct dict_connection_cmd { const struct dict_cmd_func *cmd; struct dict_connection *conn; + struct timeval start_timeval; char *reply; struct dict_iterate_context *iter; @@ -29,6 +32,10 @@ struct dict_connection_cmd { unsigned int trans_id; }; +struct dict_command_stats cmd_stats; + +static int cmd_iterate_flush(struct dict_connection_cmd *cmd); + static void dict_connection_cmd_output_more(struct dict_connection_cmd *cmd); static void dict_connection_cmd_free(struct dict_connection_cmd *cmd) @@ -67,6 +74,10 @@ static void dict_connection_cmds_flush(struct dict_connection *conn) first_cmdp = array_idx(&conn->cmds, 0); cmd = *first_cmdp; + /* we may be able to start outputting iterations now. */ + if (cmd->iter != NULL) + (void)cmd_iterate_flush(cmd); + if (cmd->reply == NULL) { /* command not finished yet */ break; @@ -78,11 +89,29 @@ static void dict_connection_cmds_flush(struct dict_connection *conn) dict_connection_unref_safe(conn); } +static void +cmd_stats_update(struct dict_connection_cmd *cmd, struct timing *timing) +{ + long long diff; + + if (!dict_settings->verbose_proctitle) + return; + + io_loop_time_refresh(); + diff = timeval_diff_usecs(&ioloop_timeval, &cmd->start_timeval); + if (diff < 0) + diff = 0; + timing_add_usecs(timing, diff); + dict_proctitle_update_later(); +} + static void cmd_lookup_callback(const struct dict_lookup_result *result, void *context) { struct dict_connection_cmd *cmd = context; + cmd_stats_update(cmd, cmd_stats.lookups); + if (result->ret > 0) { cmd->reply = i_strdup_printf("%c%s\n", DICT_PROTOCOL_REPLY_OK, str_tabescape(result->value)); @@ -142,6 +171,7 @@ static int cmd_iterate_flush(struct dict_connection_cmd *cmd) str_append_c(str, '\n'); o_stream_uncork(cmd->conn->output); + cmd_stats_update(cmd, cmd_stats.iterations); cmd->reply = i_strdup(str_c(str)); dict_connection_cmds_flush(cmd->conn); return 1; @@ -258,6 +288,8 @@ cmd_commit_finish(struct dict_connection_cmd *cmd, int ret, bool async) { char chr; + cmd_stats_update(cmd, cmd_stats.commits); + switch (ret) { case 1: chr = DICT_PROTOCOL_REPLY_OK; @@ -448,6 +480,7 @@ int dict_command_input(struct dict_connection *conn, const char *line) cmd = i_new(struct dict_connection_cmd, 1); cmd->conn = conn; cmd->cmd = cmd_func; + cmd->start_timeval = ioloop_timeval; array_append(&conn->cmds, &cmd, 1); dict_connection_ref(conn); if ((ret = cmd_func->func(cmd, line + 1)) <= 0) { @@ -485,3 +518,17 @@ void dict_connection_cmds_output_more(struct dict_connection *conn) /* cmd should be freed now */ } } + +void dict_commands_init(void) +{ + cmd_stats.lookups = timing_init(); + cmd_stats.iterations = timing_init(); + cmd_stats.commits = timing_init(); +} + +void dict_commands_deinit(void) +{ + timing_deinit(&cmd_stats.lookups); + timing_deinit(&cmd_stats.iterations); + timing_deinit(&cmd_stats.commits); +} diff --git a/src/dict/dict-commands.h b/src/dict/dict-commands.h index 03214c4e520..62200d0bbc4 100644 --- a/src/dict/dict-commands.h +++ b/src/dict/dict-commands.h @@ -3,8 +3,19 @@ struct dict_connection; +struct dict_command_stats { + struct timing *lookups; + struct timing *iterations; + struct timing *commits; +}; + +extern struct dict_command_stats cmd_stats; + int dict_command_input(struct dict_connection *conn, const char *line); void dict_connection_cmds_output_more(struct dict_connection *conn); +void dict_commands_init(void); +void dict_commands_deinit(void); + #endif diff --git a/src/dict/dict-connection.c b/src/dict/dict-connection.c index 762562a33e4..4cbffc1d787 100644 --- a/src/dict/dict-connection.c +++ b/src/dict/dict-connection.c @@ -14,9 +14,10 @@ #include -#define DICT_CONN_MAX_PENDING_COMMANDS 5 +#define DICT_CONN_MAX_PENDING_COMMANDS 1000 static struct dict_connection *dict_connections; +static unsigned int dict_connections_count = 0; static int dict_connection_parse_handshake(struct dict_connection *conn, const char *line) @@ -76,6 +77,10 @@ static int dict_connection_dict_init(struct dict_connection *conn) unsigned int i, count; const char *uri, *error; + if (!array_is_created(&dict_settings->dicts)) { + i_error("dict client: No dictionaries configured"); + return -1; + } strlist = array_get(&dict_settings->dicts, &count); for (i = 0; i < count; i += 2) { if (strcmp(strlist[i], conn->name) == 0) @@ -99,7 +104,7 @@ static int dict_connection_dict_init(struct dict_connection *conn) return 0; } -static void dict_connection_input(struct dict_connection *conn) +static void dict_connection_input_more(struct dict_connection *conn) { const char *line; int ret; @@ -107,6 +112,27 @@ static void dict_connection_input(struct dict_connection *conn) if (conn->to_input != NULL) timeout_remove(&conn->to_input); + while ((line = i_stream_next_line(conn->input)) != NULL) { + T_BEGIN { + ret = dict_command_input(conn, line); + } T_END; + if (ret < 0) { + dict_connection_destroy(conn); + break; + } + if (array_count(&conn->cmds) >= DICT_CONN_MAX_PENDING_COMMANDS) { + io_remove(&conn->io); + if (conn->to_input != NULL) + timeout_remove(&conn->to_input); + break; + } + } +} + +static void dict_connection_input(struct dict_connection *conn) +{ + const char *line; + switch (i_stream_read(conn->input)) { case 0: return; @@ -138,20 +164,7 @@ static void dict_connection_input(struct dict_connection *conn) } } - while ((line = i_stream_next_line(conn->input)) != NULL) { - T_BEGIN { - ret = dict_command_input(conn, line); - } T_END; - if (ret < 0) { - dict_connection_destroy(conn); - break; - } - if (array_count(&conn->cmds) >= DICT_CONN_MAX_PENDING_COMMANDS) { - io_remove(&conn->io); - if (conn->to_input != NULL) - timeout_remove(&conn->to_input); - } - } + dict_connection_input_more(conn); } void dict_connection_continue_input(struct dict_connection *conn) @@ -161,7 +174,7 @@ void dict_connection_continue_input(struct dict_connection *conn) conn->io = io_add(conn->fd, IO_READ, dict_connection_input, conn); if (conn->to_input == NULL) - conn->to_input = timeout_add_short(0, dict_connection_input, conn); + conn->to_input = timeout_add_short(0, dict_connection_input_more, conn); } static int dict_connection_output(struct dict_connection *conn) @@ -191,6 +204,8 @@ struct dict_connection *dict_connection_create(int fd) o_stream_set_flush_callback(conn->output, dict_connection_output, conn); conn->io = io_add(fd, IO_READ, dict_connection_input, conn); i_array_init(&conn->cmds, DICT_CONN_MAX_PENDING_COMMANDS); + + dict_connections_count++; DLLIST_PREPEND(&dict_connections, conn); return conn; } @@ -265,6 +280,9 @@ void dict_connection_destroy(struct dict_connection *conn) i_assert(!conn->destroyed); i_assert(conn->to_unref == NULL); + i_assert(dict_connections_count > 0); + dict_connections_count--; + conn->destroyed = TRUE; DLLIST_REMOVE(&dict_connections, conn); @@ -289,6 +307,11 @@ void dict_connection_destroy(struct dict_connection *conn) dict_connection_unref(conn); } +unsigned int dict_connections_current_count(void) +{ + return dict_connections_count; +} + void dict_connections_destroy_all(void) { while (dict_connections != NULL) diff --git a/src/dict/dict-connection.h b/src/dict/dict-connection.h index 87187993128..2f9149c0f05 100644 --- a/src/dict/dict-connection.h +++ b/src/dict/dict-connection.h @@ -43,6 +43,7 @@ void dict_connection_unref_safe(struct dict_connection *conn); void dict_connection_continue_input(struct dict_connection *conn); +unsigned int dict_connections_current_count(void); void dict_connections_destroy_all(void); #endif diff --git a/src/dict/dict-settings.c b/src/dict/dict-settings.c index 53a27e3c179..6b36cac62bb 100644 --- a/src/dict/dict-settings.c +++ b/src/dict/dict-settings.c @@ -86,6 +86,8 @@ struct service_settings dict_async_service_settings = { static const struct setting_define dict_setting_defines[] = { DEF(SET_STR, base_dir), + DEF(SET_BOOL, verbose_proctitle), + DEF(SET_STR, dict_db_config), { SET_STRLIST, "dict", offsetof(struct dict_server_settings, dicts), NULL }, @@ -94,6 +96,8 @@ static const struct setting_define dict_setting_defines[] = { const struct dict_server_settings dict_default_settings = { .base_dir = PKG_RUNDIR, + .verbose_proctitle = FALSE, + .dict_db_config = "", .dicts = ARRAY_INIT }; diff --git a/src/dict/dict-settings.h b/src/dict/dict-settings.h index 0068ef04a46..3c37589e9be 100644 --- a/src/dict/dict-settings.h +++ b/src/dict/dict-settings.h @@ -3,6 +3,8 @@ struct dict_server_settings { const char *base_dir; + bool verbose_proctitle; + const char *dict_db_config; ARRAY(const char *) dicts; }; diff --git a/src/dict/main.c b/src/dict/main.c index e6c945eb918..f1d89fe27f9 100644 --- a/src/dict/main.c +++ b/src/dict/main.c @@ -2,8 +2,12 @@ #include "lib.h" #include "restrict-access.h" +#include "ioloop.h" #include "randgen.h" +#include "str.h" #include "hostpid.h" +#include "timing.h" +#include "process-title.h" #include "env-util.h" #include "module-dir.h" #include "master-service.h" @@ -11,10 +15,54 @@ #include "sql-api.h" #include "dict.h" #include "dict-client.h" +#include "dict-commands.h" #include "dict-connection.h" #include "dict-settings.h" +#include "main.h" static struct module *modules; +static struct timeout *to_proctitle; +static bool proctitle_updated; + +static void +add_timing_string(string_t *str, struct timing *timing, const char *name) +{ + str_printfa(str, ", %u %s:%llu/%llu/%llu/%llu", + timing_get_count(timing), name, + (unsigned long long)timing_get_min(timing)/1000, + (unsigned long long)timing_get_avg(timing)/1000, + (unsigned long long)timing_get_95th(timing)/1000, + (unsigned long long)timing_get_max(timing)/1000); + timing_reset(timing); +} + +static void dict_proctitle_update(void *context ATTR_UNUSED) +{ + string_t *str = t_str_new(128); + + if (!proctitle_updated) + timeout_remove(&to_proctitle); + + str_printfa(str, "[%u clients", dict_connections_current_count()); + + add_timing_string(str, cmd_stats.lookups, "lookups"); + add_timing_string(str, cmd_stats.iterations, "iters"); + add_timing_string(str, cmd_stats.commits, "commits"); + str_append_c(str, ']'); + + process_title_set(str_c(str)); + proctitle_updated = FALSE; +} + +void dict_proctitle_update_later(void) +{ + if (!dict_settings->verbose_proctitle) + return; + + if (to_proctitle == NULL) + to_proctitle = timeout_add(1000, dict_proctitle_update, NULL); + proctitle_updated = TRUE; +} static void dict_die(void) { @@ -68,12 +116,17 @@ static void main_init(void) /* Register only after loading modules. They may contain SQL drivers, which we'll need to register. */ dict_drivers_register_all(); + dict_commands_init(); } static void main_deinit(void) { + if (to_proctitle != NULL) + timeout_remove(&to_proctitle); + dict_connections_destroy_all(); dict_drivers_unregister_all(); + dict_commands_deinit(); module_dir_unload(&modules); @@ -83,8 +136,7 @@ static void main_deinit(void) int main(int argc, char *argv[]) { - const enum master_service_flags service_flags = - MASTER_SERVICE_FLAG_UPDATE_PROCTITLE; + const enum master_service_flags service_flags = 0; const struct setting_parser_info *set_roots[] = { &dict_setting_parser_info, NULL diff --git a/src/dict/main.h b/src/dict/main.h new file mode 100644 index 00000000000..e505e134c89 --- /dev/null +++ b/src/dict/main.h @@ -0,0 +1,6 @@ +#ifndef MAIN_H +#define MAIN_H + +void dict_proctitle_update_later(void); + +#endif diff --git a/src/director/director-connection.c b/src/director/director-connection.c index a727658efd7..d8b58d3c563 100644 --- a/src/director/director-connection.c +++ b/src/director/director-connection.c @@ -927,6 +927,11 @@ director_cmd_host_int(struct director_connection *conn, const char *const *args, str_printfa(str, "director(%s): Host %s is being updated before previous update had finished (", conn->name, net_ip2addr(&host->ip)); + if (host->down != down && + host->last_updown_change > last_updown_change) { + /* our host has a newer change. preserve it. */ + down = host->down; + } if (host->down != down) { if (host->down) str_append(str, "down -> up"); @@ -942,10 +947,6 @@ director_cmd_host_int(struct director_connection *conn, const char *const *args, str_append(str, ") - "); vhost_count = I_MIN(vhost_count, host->vhost_count); - if (host->down != down) { - if (host->last_updown_change <= last_updown_change) - down = host->last_updown_change; - } last_updown_change = I_MAX(last_updown_change, host->last_updown_change); str_printfa(str, "setting to state=%s vhosts=%u", @@ -1238,7 +1239,7 @@ director_connection_handle_handshake(struct director_connection *conn, if (conn->minor_version < DIRECTOR_VERSION_TAGS_V2 && mail_hosts_have_tags(conn->dir->mail_hosts)) { i_error("director(%s): Director version supports incompatible tags", conn->name); - return FALSE; + return -1; } conn->version_received = TRUE; if (conn->done_pending) { @@ -2051,9 +2052,10 @@ void director_connection_send(struct director_connection *conn, } T_END; ret = o_stream_send(conn->output, data, len); if (ret != (off_t)len) { - if (ret < 0) - i_error("director(%s): write() failed: %m", conn->name); - else { + if (ret < 0) { + i_error("director(%s): write() failed: %s", conn->name, + o_stream_get_error(conn->output)); + } else { i_error("director(%s): Output buffer full, " "disconnecting", conn->name); } diff --git a/src/doveadm/Makefile.am b/src/doveadm/Makefile.am index cfdaf2152c2..5195355d4dc 100644 --- a/src/doveadm/Makefile.am +++ b/src/doveadm/Makefile.am @@ -20,6 +20,7 @@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib-imap \ -I$(top_srcdir)/src/lib-index \ -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-imap-storage \ -I$(top_srcdir)/src/lib-http \ -I$(top_srcdir)/src/auth \ -DMODULEDIR=\""$(moduledir)"\" \ @@ -75,6 +76,7 @@ doveadm_common_cmds = \ doveadm-penalty.c \ doveadm-proxy.c \ doveadm-replicator.c \ + doveadm-service.c \ doveadm-sis.c \ doveadm-stats.c \ doveadm-who.c @@ -135,6 +137,7 @@ doveadm_SOURCES = \ doveadm_server_SOURCES = \ $(common) \ + doveadm-auth-server.c \ client-connection.c \ client-connection-http.c \ doveadm-print-server.c \ diff --git a/src/doveadm/client-connection-http.c b/src/doveadm/client-connection-http.c index 7fa530ba494..63716faff9e 100644 --- a/src/doveadm/client-connection-http.c +++ b/src/doveadm/client-connection-http.c @@ -189,7 +189,13 @@ static void doveadm_http_server_json_success(void *context, struct istream *resu string_t *escaped; escaped = str_new(conn->client.pool, 10); o_stream_nsend_str(conn->client.output,"[\"doveadmResponse\","); - o_stream_send_istream(conn->client.output, result); + if (o_stream_send_istream(conn->client.output, result) < 0) { + if (conn->client.output->stream_errno != 0) { + i_fatal("write(%s) failed: %s", o_stream_get_name(conn->client.output), o_stream_get_error(conn->client.output)); + } else if (result->stream_errno != 0) { + i_fatal("read(%s) failed: %s", i_stream_get_name(result), i_stream_get_error(result)); + } else i_unreached(); /* either it's output or input error */ + } o_stream_nsend_str(conn->client.output,",\""); if (conn->method_id != NULL) { json_append_escaped(escaped, conn->method_id); @@ -323,8 +329,11 @@ doveadm_http_server_command_execute(struct client_connection_http *conn) // create iostream doveadm_print_ostream = iostream_temp_create("/tmp/doveadm.", 0); + cctx.cmd = conn->cmd; + + if ((cctx.cmd->flags & CMD_FLAG_NO_PRINT) == 0) + doveadm_print_init(DOVEADM_PRINT_TYPE_JSON); - doveadm_print_init(DOVEADM_PRINT_TYPE_JSON); /* then call it */ doveadm_cmd_params_null_terminate_arrays(&conn->pargv); cctx.argv = array_get(&conn->pargv, (unsigned int*)&cctx.argc); @@ -332,7 +341,6 @@ doveadm_http_server_command_execute(struct client_connection_http *conn) lib_signals_reset_ioloop(); doveadm_exit_code = 0; - cctx.cmd = conn->cmd; cctx.cli = FALSE; cctx.local_ip = conn->client.local_ip; cctx.local_port = conn->client.local_port; @@ -351,7 +359,8 @@ doveadm_http_server_command_execute(struct client_connection_http *conn) io_loop_set_current(ioloop); io_loop_destroy(&ioloop); - doveadm_print_deinit(); + if ((cctx.cmd->flags & CMD_FLAG_NO_PRINT) == 0) + doveadm_print_deinit(); if (o_stream_nfinish(doveadm_print_ostream)<0) { i_info("Error writing output in command %s: %s", conn->cmd->name, @@ -650,13 +659,13 @@ doveadm_http_server_authorize_request(struct client_connection_http *conn) string_t *b64_value = str_new(conn->client.pool, 32); char *value = p_strdup_printf(conn->client.pool, "doveadm:%s", conn->client.set->doveadm_password); base64_encode(value, strlen(value), b64_value); - if (strcmp(creds.data, str_c(b64_value)) == 0) auth = TRUE; + if (creds.data != NULL && strcmp(creds.data, str_c(b64_value)) == 0) auth = TRUE; else i_error("Invalid authentication attempt to HTTP API"); } else if (strcasecmp(creds.scheme, "X-Dovecot-API") == 0 && doveadm_settings->doveadm_api_key[0] != '\0') { string_t *b64_value = str_new(conn->client.pool, 32); base64_encode(doveadm_settings->doveadm_api_key, strlen(doveadm_settings->doveadm_api_key), b64_value); - if (strcmp(creds.data, str_c(b64_value)) == 0) auth = TRUE; + if (creds.data != NULL && strcmp(creds.data, str_c(b64_value)) == 0) auth = TRUE; else i_error("Invalid authentication attempt to HTTP API"); } else i_error("Unsupported authentication scheme to HTTP API"); @@ -751,7 +760,7 @@ static void doveadm_http_server_send_response(void *context) } static const struct http_server_settings http_server_set = { - .max_client_idle_time_msecs = 5000, + .max_client_idle_time_msecs = 0, .max_pipelined_requests = 0 }; diff --git a/src/doveadm/doveadm-auth-server.c b/src/doveadm/doveadm-auth-server.c new file mode 100644 index 00000000000..eb1d9aa9fca --- /dev/null +++ b/src/doveadm/doveadm-auth-server.c @@ -0,0 +1,503 @@ +/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "str.h" +#include "var-expand.h" +#include "wildcard-match.h" +#include "settings-parser.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "auth-client.h" +#include "auth-master.h" +#include "auth-server-connection.h" +#include "master-auth.h" +#include "master-login-auth.h" +#include "mail-storage-service.h" +#include "mail-user.h" +#include "ostream.h" +#include "json-parser.h" +#include "doveadm.h" +#include "doveadm-print.h" + +#include +#include + +struct authtest_input { + pool_t pool; + const char *username; + const char *master_user; + const char *password; + struct auth_user_info info; + bool success; + + struct auth_client_request *request; + struct master_auth_request master_auth_req; + + unsigned int auth_id; + unsigned int auth_pid; + const char *auth_cookie; + +}; + +static struct auth_master_connection * +doveadm_get_auth_master_conn(const char *auth_socket_path) +{ + enum auth_master_flags flags = 0; + + if (doveadm_debug) + flags |= AUTH_MASTER_FLAG_DEBUG; + return auth_master_init(auth_socket_path, flags); +} + +static int +cmd_user_input(struct auth_master_connection *conn, + const struct authtest_input *input, + const char *show_field, bool userdb) +{ + const char *lookup_name = userdb ? "userdb lookup" : "passdb lookup"; + pool_t pool; + const char *updated_username = NULL, *const *fields, *p; + int ret; + + pool = pool_alloconly_create("auth master lookup", 1024); + + if (userdb) { + ret = auth_master_user_lookup(conn, input->username, &input->info, + pool, &updated_username, &fields); + } else { + ret = auth_master_pass_lookup(conn, input->username, &input->info, + pool, &fields); + } + if (ret < 0) { + const char *msg; + if (fields[0] == NULL) { + msg = t_strdup_printf("\"error\":\"%s failed\"", + lookup_name); + } else { + msg = t_strdup_printf("\"error\":\"%s failed: %s\"", + lookup_name, + fields[0]); + } + o_stream_nsend_str(doveadm_print_ostream, msg); + ret = -1; + } else if (ret == 0) { + o_stream_nsend_str(doveadm_print_ostream, + t_strdup_printf("\"error\":\"%s: user doesn't exist\"", + lookup_name)); + } else if (show_field != NULL) { + unsigned int show_field_len = strlen(show_field); + string_t *json_field = t_str_new(show_field_len+1); + json_append_escaped(json_field, show_field); + o_stream_nsend_str(doveadm_print_ostream, t_strdup_printf("\"%s\":", str_c(json_field))); + for (; *fields; fields++) { + if (strncmp(*fields, show_field, show_field_len) == 0 && + (*fields)[show_field_len] == '=') { + string_t *jsonval = t_str_new(32); + json_append_escaped(jsonval, *fields + show_field_len + 1); + o_stream_nsend_str(doveadm_print_ostream, "\""); + o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval)); + o_stream_nsend_str(doveadm_print_ostream, "\""); + } + } + } else { + string_t *jsonval = t_str_new(64); + o_stream_nsend_str(doveadm_print_ostream, "\"source\":\""); + o_stream_nsend_str(doveadm_print_ostream, userdb ? "userdb\"" : "passdb\""); + + if (updated_username != NULL) { + o_stream_nsend_str(doveadm_print_ostream, ",\"updated_username\":\""); + str_truncate(jsonval, 0); + json_append_escaped(jsonval, updated_username); + o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval)); + o_stream_nsend_str(doveadm_print_ostream, "\""); + } + for (; *fields; fields++) { + const char *field = *fields; + if (*field == '\0') continue; + p = strchr(*fields, '='); + str_truncate(jsonval, 0); + if (p != NULL) { + field = t_strcut(*fields, '='); + } + str_truncate(jsonval, 0); + json_append_escaped(jsonval, field); + o_stream_nsend_str(doveadm_print_ostream, ",\""); + o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval)); + o_stream_nsend_str(doveadm_print_ostream, "\":"); + if (p != NULL) { + str_truncate(jsonval, 0); + json_append_escaped(jsonval, p+1); + o_stream_nsend_str(doveadm_print_ostream, "\""); + o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval)); + o_stream_nsend_str(doveadm_print_ostream, "\""); + } else { + o_stream_nsend_str(doveadm_print_ostream, "true"); + } + } + } + return ret; +} + +static void auth_user_info_parse(struct auth_user_info *info, const char *arg) +{ + if (strncmp(arg, "service=", 8) == 0) + info->service = arg + 8; + else if (strncmp(arg, "lip=", 4) == 0) { + if (net_addr2ip(arg + 4, &info->local_ip) < 0) + i_fatal("lip: Invalid ip"); + } else if (strncmp(arg, "rip=", 4) == 0) { + if (net_addr2ip(arg + 4, &info->remote_ip) < 0) + i_fatal("rip: Invalid ip"); + } else if (strncmp(arg, "lport=", 6) == 0) { + if (net_str2port(arg + 6, &info->local_port) < 0) + i_fatal("lport: Invalid port number"); + } else if (strncmp(arg, "rport=", 6) == 0) { + if (net_str2port(arg + 6, &info->remote_port) < 0) + i_fatal("rport: Invalid port number"); + } else { + i_fatal("Unknown -x argument: %s", arg); + } +} + +static void +cmd_user_list(struct auth_master_connection *conn, + const struct authtest_input *input, + char *const *users) +{ + struct auth_master_user_list_ctx *ctx; + const char *username, *user_mask = "*"; + unsigned int i; + + if (users[0] != NULL && users[1] == NULL) + user_mask = users[0]; + + ctx = auth_master_user_list_init(conn, user_mask, &input->info); + while ((username = auth_master_user_list_next(ctx)) != NULL) { + for (i = 0; users[i] != NULL; i++) { + if (wildcard_match_icase(username, users[i])) + break; + } + if (users[i] != NULL) + printf("%s\n", username); + } + if (auth_master_user_list_deinit(&ctx) < 0) + i_fatal("user listing failed"); +} + +static void cmd_auth_cache_flush(int argc, char *argv[]) +{ + const char *master_socket_path = NULL; + struct auth_master_connection *conn; + unsigned int count; + int c; + + while ((c = getopt(argc, argv, "a:")) > 0) { + switch (c) { + case 'a': + master_socket_path = optarg; + break; + default: + doveadm_exit_code = EX_USAGE; + return; + } + } + argv += optind; + + if (master_socket_path == NULL) { + master_socket_path = t_strconcat(doveadm_settings->base_dir, + "/auth-master", NULL); + } + + conn = doveadm_get_auth_master_conn(master_socket_path); + if (auth_master_cache_flush(conn, (void *)argv, &count) < 0) { + i_error("Cache flush failed"); + doveadm_exit_code = EX_TEMPFAIL; + } else { + doveadm_print_init("formatted"); + doveadm_print_formatted_set_format("%{entries} cache entries flushed\n"); + doveadm_print_header_simple("entries"); + doveadm_print_num(count); + } + auth_master_deinit(&conn); +} + +static void cmd_user_mail_input_field(const char *key, const char *value, + const char *show_field) +{ + string_t *jvalue = t_str_new(128); + if (show_field != NULL && strcmp(show_field, key) != 0) return; + json_append_escaped(jvalue, key); + o_stream_nsend_str(doveadm_print_ostream, "\""); + o_stream_nsend_str(doveadm_print_ostream, str_c(jvalue)); + o_stream_nsend_str(doveadm_print_ostream, "\":\""); + str_truncate(jvalue, 0); + json_append_escaped(jvalue, value); + o_stream_nsend_str(doveadm_print_ostream, str_c(jvalue)); + o_stream_nsend_str(doveadm_print_ostream, "\""); +} + +static void +cmd_user_mail_print_fields(const struct authtest_input *input, + struct mail_user *user, + const char *const *userdb_fields, + const char *show_field) +{ + const struct mail_storage_settings *mail_set; + const char *key, *value; + unsigned int i; + + if (strcmp(input->username, user->username) != 0) { + cmd_user_mail_input_field("user", user->username, show_field); + o_stream_nsend_str(doveadm_print_ostream, ","); + } + cmd_user_mail_input_field("uid", user->set->mail_uid, show_field); + o_stream_nsend_str(doveadm_print_ostream, ","); + cmd_user_mail_input_field("gid", user->set->mail_gid, show_field); + o_stream_nsend_str(doveadm_print_ostream, ","); + cmd_user_mail_input_field("home", user->set->mail_home, show_field); + + mail_set = mail_user_set_get_storage_set(user); + o_stream_nsend_str(doveadm_print_ostream, ","); + cmd_user_mail_input_field("mail", mail_set->mail_location, show_field); + + if (userdb_fields != NULL) { + for (i = 0; userdb_fields[i] != NULL; i++) { + value = strchr(userdb_fields[i], '='); + if (value != NULL) + key = t_strdup_until(userdb_fields[i], value++); + else { + key = userdb_fields[i]; + value = ""; + } + if (strcmp(key, "uid") != 0 && + strcmp(key, "gid") != 0 && + strcmp(key, "home") != 0 && + strcmp(key, "mail") != 0 && + *key != '\0') { + o_stream_nsend_str(doveadm_print_ostream, ","); + cmd_user_mail_input_field(key, value, show_field); + } + } + } +} + +static int +cmd_user_mail_input(struct mail_storage_service_ctx *storage_service, + const struct authtest_input *input, + const char *show_field, const char *expand_field) +{ + struct mail_storage_service_input service_input; + struct mail_storage_service_user *service_user; + struct mail_user *user; + const char *error, *const *userdb_fields; + pool_t pool; + int ret; + + memset(&service_input, 0, sizeof(service_input)); + service_input.module = "mail"; + service_input.service = input->info.service; + service_input.username = input->username; + service_input.local_ip = input->info.local_ip; + service_input.local_port = input->info.local_port; + service_input.remote_ip = input->info.remote_ip; + service_input.remote_port = input->info.remote_port; + service_input.debug = input->info.debug; + + pool = pool_alloconly_create("userdb fields", 1024); + mail_storage_service_save_userdb_fields(storage_service, pool, + &userdb_fields); + + if ((ret = mail_storage_service_lookup_next(storage_service, &service_input, + &service_user, &user, + &error)) <= 0) { + pool_unref(&pool); + if (ret < 0) + return -1; + string_t *username = t_str_new(32); + json_append_escaped(username, input->username); + o_stream_nsend_str(doveadm_print_ostream, + t_strdup_printf("\"error\":\"userdb lookup: user %s doesn't exist\"", str_c(username)) + ); + return 0; + } + + if (expand_field == NULL) + cmd_user_mail_print_fields(input, user, userdb_fields, show_field); + else { + string_t *str = t_str_new(128); + var_expand_with_funcs(str, expand_field, + mail_user_var_expand_table(user), + mail_user_var_expand_func_table, user); + string_t *value = t_str_new(128); + json_append_escaped(value, expand_field); + o_stream_nsend_str(doveadm_print_ostream, "\""); + o_stream_nsend_str(doveadm_print_ostream, str_c(value)); + o_stream_nsend_str(doveadm_print_ostream, "\":\""); + str_truncate(value, 0); + json_append_escaped(value, str_c(str)); + o_stream_nsend_str(doveadm_print_ostream, str_c(value)); + o_stream_nsend_str(doveadm_print_ostream, "\""); + + } + + mail_user_unref(&user); + mail_storage_service_user_free(&service_user); + pool_unref(&pool); + return 1; +} + +static void cmd_user_ver2(struct doveadm_cmd_context *cctx) +{ + const char * const *optval; + + const char *auth_socket_path = NULL; + struct auth_master_connection *conn; + struct authtest_input input; + const char *show_field = NULL, *expand_field = NULL; + struct mail_storage_service_ctx *storage_service = NULL; + bool have_wildcards, userdb_only = FALSE, first = TRUE; + int ret; + + if (!doveadm_cmd_param_str(cctx, "socket-path", &auth_socket_path)) + auth_socket_path = doveadm_settings->auth_socket_path; + + (void)doveadm_cmd_param_str(cctx, "expand-field", &expand_field); + (void)doveadm_cmd_param_str(cctx, "field", &show_field); + (void)doveadm_cmd_param_bool(cctx, "userdb-only", &userdb_only); + + memset(&input, 0, sizeof(input)); + if (doveadm_cmd_param_array(cctx, "auth-info", &optval)) + for(;*optval != NULL; optval++) + auth_user_info_parse(&input.info, *optval); + + if (!doveadm_cmd_param_array(cctx, "user-mask", &optval)) { + doveadm_exit_code = EX_USAGE; + i_error("No user(s) specified"); + return; + } + + if (expand_field != NULL && userdb_only) { + i_error("-e can't be used with -u"); + doveadm_exit_code = EX_USAGE; + return; + } + if (expand_field != NULL && show_field != NULL) { + i_error("-e can't be used with -f"); + doveadm_exit_code = EX_USAGE; + return; + } + + conn = doveadm_get_auth_master_conn(auth_socket_path); + + have_wildcards = FALSE; + + for(const char *const *val = optval; *val != NULL; val++) { + if (strchr(*val, '*') != NULL || + strchr(*val, '?') != NULL) { + have_wildcards = TRUE; + break; + } + } + + if (have_wildcards) { + cmd_user_list(conn, &input, (char*const*)optval); + auth_master_deinit(&conn); + return; + } + + if (!userdb_only) { + storage_service = mail_storage_service_init(master_service, NULL, + MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP | + MAIL_STORAGE_SERVICE_FLAG_NO_CHDIR | + MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT | + MAIL_STORAGE_SERVICE_FLAG_NO_PLUGINS | + MAIL_STORAGE_SERVICE_FLAG_NO_NAMESPACES | + MAIL_STORAGE_SERVICE_FLAG_NO_RESTRICT_ACCESS); + mail_storage_service_set_auth_conn(storage_service, conn); + conn = NULL; + } + + string_t *json = t_str_new(64); + o_stream_nsend_str(doveadm_print_ostream, "{"); + + input.info.local_ip = cctx->local_ip; + input.info.local_port = cctx->local_port; + input.info.remote_ip = cctx->remote_ip; + input.info.remote_port = cctx->remote_port; + + for(const char *const *val = optval; *val != NULL; val++) { + str_truncate(json, 0); + json_append_escaped(json, *val); + + input.username = *val; + if (first) + first = FALSE; + else + o_stream_nsend_str(doveadm_print_ostream, ","); + + o_stream_nsend_str(doveadm_print_ostream, "\""); + o_stream_nsend_str(doveadm_print_ostream, str_c(json)); + o_stream_nsend_str(doveadm_print_ostream, "\""); + o_stream_nsend_str(doveadm_print_ostream, ":{"); + + ret = !userdb_only ? + cmd_user_mail_input(storage_service, &input, show_field, expand_field) : + cmd_user_input(conn, &input, show_field, TRUE); + + o_stream_nsend_str(doveadm_print_ostream, "}"); + + switch (ret) { + case -1: + doveadm_exit_code = EX_TEMPFAIL; + break; + case 0: + doveadm_exit_code = EX_NOUSER; + break; + } + } + + o_stream_nsend_str(doveadm_print_ostream,"}"); + + if (storage_service != NULL) + mail_storage_service_deinit(&storage_service); + if (conn != NULL) + auth_master_deinit(&conn); +} + +static +struct doveadm_cmd_ver2 doveadm_cmd_auth_server[] = { +{ + .name = "auth cache flush", + .old_cmd = cmd_auth_cache_flush, + .usage = "[-a ] [ [...]]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "user", + .cmd = cmd_user_ver2, + .usage = "[-a ] [-x ] [-f field] [-e ] [-u] [...]", + .flags = CMD_FLAG_NO_PRINT, +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('x', "auth-info", CMD_PARAM_ARRAY, 0) +DOVEADM_CMD_PARAM('f', "field", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('e', "expand-field", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('u', "userdb-only", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "user-mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +} +}; + +void doveadm_register_auth_server_commands(void) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_auth_server); i++) { + doveadm_cmd_register_ver2(&doveadm_cmd_auth_server[i]); + } +} diff --git a/src/doveadm/doveadm-auth.c b/src/doveadm/doveadm-auth.c index 55aa9b15533..338deb6d289 100644 --- a/src/doveadm/doveadm-auth.c +++ b/src/doveadm/doveadm-auth.c @@ -7,6 +7,7 @@ #include "base64.h" #include "hex-binary.h" #include "str.h" +#include "var-expand.h" #include "wildcard-match.h" #include "settings-parser.h" #include "master-service.h" @@ -489,16 +490,52 @@ static void cmd_user_mail_input_field(const char *key, const char *value, } } -static int cmd_user_mail_input(struct mail_storage_service_ctx *storage_service, - const struct authtest_input *input, - const char *show_field) +static void +cmd_user_mail_print_fields(const struct authtest_input *input, + struct mail_user *user, + const char *const *userdb_fields, + const char *show_field) +{ + const struct mail_storage_settings *mail_set; + const char *key, *value; + unsigned int i; + + if (strcmp(input->username, user->username) != 0) + cmd_user_mail_input_field("user", user->username, show_field); + cmd_user_mail_input_field("uid", user->set->mail_uid, show_field); + cmd_user_mail_input_field("gid", user->set->mail_gid, show_field); + cmd_user_mail_input_field("home", user->set->mail_home, show_field); + + mail_set = mail_user_set_get_storage_set(user); + cmd_user_mail_input_field("mail", mail_set->mail_location, show_field); + + if (userdb_fields != NULL) { + for (i = 0; userdb_fields[i] != NULL; i++) { + value = strchr(userdb_fields[i], '='); + if (value != NULL) + key = t_strdup_until(userdb_fields[i], value++); + else { + key = userdb_fields[i]; + value = ""; + } + if (strcmp(key, "uid") != 0 && + strcmp(key, "gid") != 0 && + strcmp(key, "home") != 0 && + strcmp(key, "mail") != 0) + cmd_user_mail_input_field(key, value, show_field); + } + } +} + +static int +cmd_user_mail_input(struct mail_storage_service_ctx *storage_service, + const struct authtest_input *input, + const char *show_field, const char *expand_field) { struct mail_storage_service_input service_input; struct mail_storage_service_user *service_user; struct mail_user *user; - const struct mail_storage_settings *mail_set; - const char *key, *value, *error, *const *userdb_fields; - unsigned int i; + const char *error, *const *userdb_fields; pool_t pool; int ret; @@ -522,36 +559,20 @@ static int cmd_user_mail_input(struct mail_storage_service_ctx *storage_service, pool_unref(&pool); if (ret < 0) return -1; - fprintf(show_field == NULL ? stdout : stderr, + fprintf(show_field == NULL && expand_field == NULL ? stdout : stderr, "userdb lookup: user %s doesn't exist\n", input->username); return 0; } - if (strcmp(input->username, user->username) != 0) - cmd_user_mail_input_field("user", user->username, show_field); - cmd_user_mail_input_field("uid", user->set->mail_uid, show_field); - cmd_user_mail_input_field("gid", user->set->mail_gid, show_field); - cmd_user_mail_input_field("home", user->set->mail_home, show_field); - - mail_set = mail_user_set_get_storage_set(user); - cmd_user_mail_input_field("mail", mail_set->mail_location, show_field); - - if (userdb_fields != NULL) { - for (i = 0; userdb_fields[i] != NULL; i++) { - value = strchr(userdb_fields[i], '='); - if (value != NULL) - key = t_strdup_until(userdb_fields[i], value++); - else { - key = userdb_fields[i]; - value = ""; - } - if (strcmp(key, "uid") != 0 && - strcmp(key, "gid") != 0 && - strcmp(key, "home") != 0 && - strcmp(key, "mail") != 0) - cmd_user_mail_input_field(key, value, show_field); - } + if (expand_field == NULL) + cmd_user_mail_print_fields(input, user, userdb_fields, show_field); + else { + string_t *str = t_str_new(128); + var_expand_with_funcs(str, expand_field, + mail_user_var_expand_table(user), + mail_user_var_expand_func_table, user); + printf("%s\n", str_c(str)); } mail_user_unref(&user); @@ -565,18 +586,21 @@ static void cmd_user(int argc, char *argv[]) const char *auth_socket_path = doveadm_settings->auth_socket_path; struct auth_master_connection *conn; struct authtest_input input; - const char *show_field = NULL; + const char *show_field = NULL, *expand_field = NULL; struct mail_storage_service_ctx *storage_service = NULL; unsigned int i; bool have_wildcards, userdb_only = FALSE, first = TRUE; int c, ret; authtest_input_init(&input); - while ((c = getopt(argc, argv, "a:f:ux:")) > 0) { + while ((c = getopt(argc, argv, "a:e:f:ux:")) > 0) { switch (c) { case 'a': auth_socket_path = optarg; break; + case 'e': + expand_field = optarg; + break; case 'f': show_field = optarg; break; @@ -591,6 +615,17 @@ static void cmd_user(int argc, char *argv[]) } } + if (expand_field != NULL && userdb_only) { + i_error("-e can't be used with -u"); + doveadm_exit_code = EX_USAGE; + return; + } + if (expand_field != NULL && show_field != NULL) { + i_error("-e can't be used with -f"); + doveadm_exit_code = EX_USAGE; + return; + } + if (optind == argc) auth_cmd_help(cmd_user); @@ -621,7 +656,7 @@ static void cmd_user(int argc, char *argv[]) MAIL_STORAGE_SERVICE_FLAG_NO_RESTRICT_ACCESS); mail_storage_service_set_auth_conn(storage_service, conn); conn = NULL; - if (show_field == NULL) { + if (show_field == NULL && expand_field == NULL) { doveadm_print_init(DOVEADM_PRINT_TYPE_TAB); doveadm_print_header_simple("field"); doveadm_print_header_simple("value"); @@ -635,7 +670,7 @@ static void cmd_user(int argc, char *argv[]) putchar('\n'); ret = !userdb_only ? - cmd_user_mail_input(storage_service, &input, show_field) : + cmd_user_mail_input(storage_service, &input, show_field, expand_field) : cmd_user_input(conn, &input, show_field, TRUE); switch (ret) { case -1: @@ -662,7 +697,7 @@ struct doveadm_cmd doveadm_cmd_auth[] = { { cmd_auth_cache_flush, "auth cache flush", "[-a ] [ [...]]" }, { cmd_user, "user", - "[-a ] [-x ] [-f field] [-u] [...]" } + "[-a ] [-x ] [-f field] [-e ] [-u] [...]" } }; static void auth_cmd_help(doveadm_command_t *cmd) diff --git a/src/doveadm/doveadm-cmd.c b/src/doveadm/doveadm-cmd.c index 8d9ff5092ee..3c2da477bc4 100644 --- a/src/doveadm/doveadm-cmd.c +++ b/src/doveadm/doveadm-cmd.c @@ -19,6 +19,7 @@ static struct doveadm_cmd *doveadm_commands[] = { }; static struct doveadm_cmd_ver2 *doveadm_commands_ver2[] = { + &doveadm_cmd_service_stop_ver2, &doveadm_cmd_stop_ver2, &doveadm_cmd_reload_ver2, &doveadm_cmd_stats_dump_ver2, @@ -173,7 +174,6 @@ void doveadm_cmds_init(void) for (i = 0; i < N_ELEMENTS(doveadm_commands_ver2); i++) doveadm_cmd_register_ver2(doveadm_commands_ver2[i]); - doveadm_register_auth_commands(); doveadm_register_director_commands(); doveadm_register_instance_commands(); doveadm_register_mount_commands(); @@ -378,18 +378,17 @@ doveadm_build_options(const struct doveadm_cmd_param par[], { for(size_t i=0; par[i].name != NULL; i++) { struct option longopt; + + memset(&longopt, 0, sizeof(longopt)); longopt.name = par[i].name; - longopt.flag = 0; - longopt.val = 0; if (par[i].short_opt != '\0') { longopt.val = par[i].short_opt; str_append_c(shortopts, par[i].short_opt); if (par[i].type != CMD_PARAM_BOOL) str_append_c(shortopts, ':'); - } else { - if (par[i].type != CMD_PARAM_BOOL) longopt.has_arg = 1; - else longopt.has_arg = 0; } + if (par[i].type != CMD_PARAM_BOOL) + longopt.has_arg = 1; array_append(longopts, &longopt, 1); } array_append_zero(longopts); diff --git a/src/doveadm/doveadm-cmd.h b/src/doveadm/doveadm-cmd.h index 6e2d2b878a1..19a46cdedf4 100644 --- a/src/doveadm/doveadm-cmd.h +++ b/src/doveadm/doveadm-cmd.h @@ -30,7 +30,8 @@ typedef enum { typedef enum { CMD_FLAG_NONE = 0x0, - CMD_FLAG_HIDDEN = 0x1 + CMD_FLAG_HIDDEN = 0x1, + CMD_FLAG_NO_PRINT = 0x2, } doveadm_cmd_flag_t; struct doveadm_cmd_param { @@ -100,6 +101,7 @@ doveadm_cmd_find_with_args(const char *cmd_name, int *argc, const char *const *argv[]); void doveadm_register_auth_commands(void); +void doveadm_register_auth_server_commands(void); void doveadm_register_director_commands(void); void doveadm_register_proxy_commands(void); void doveadm_register_log_commands(void); @@ -144,6 +146,7 @@ bool doveadm_cmd_param_istream(const struct doveadm_cmd_context *cctx, void doveadm_cmd_params_clean(ARRAY_TYPE(doveadm_cmd_param_arr_t) *pargv); void doveadm_cmd_params_null_terminate_arrays(ARRAY_TYPE(doveadm_cmd_param_arr_t) *pargv); +extern struct doveadm_cmd_ver2 doveadm_cmd_service_stop_ver2; extern struct doveadm_cmd_ver2 doveadm_cmd_stop_ver2; extern struct doveadm_cmd_ver2 doveadm_cmd_reload_ver2; extern struct doveadm_cmd_ver2 doveadm_cmd_stats_reset_ver2; diff --git a/src/doveadm/doveadm-director.c b/src/doveadm/doveadm-director.c index 76accd25661..46b2cae2ee8 100644 --- a/src/doveadm/doveadm-director.c +++ b/src/doveadm/doveadm-director.c @@ -21,6 +21,13 @@ struct director_context { const char *socket_path; const char *users_path; const char *tag; + const char *user; + const char *host; + const char *ip; + const char *port; + const char *vhost_count; + + struct istream *users_input; struct istream *input; bool explicit_socket_path; bool hash_map, user_map, force_flush; @@ -33,10 +40,9 @@ struct user_list { HASH_TABLE_DEFINE_TYPE(user_list, void *, struct user_list *); -extern struct doveadm_cmd doveadm_cmd_director[]; - -static void director_cmd_help(doveadm_command_t *cmd) ATTR_NORETURN; - +static void director_cmd_help(const struct doveadm_cmd_ver2 *); +static int director_get_host(const char *host, struct ip_addr **ips_r, + unsigned int *ips_count_r) ATTR_WARN_UNUSED_RESULT; static void director_send(struct director_context *ctx, const char *data) { @@ -60,11 +66,12 @@ static void director_connect(struct director_context *ctx) line = i_stream_read_next_line(ctx->input); alarm(0); if (line == NULL) { - if (ctx->input->stream_errno != 0) - i_fatal("read(%s) failed: %m", ctx->socket_path); - else if (ctx->input->eof) + if (ctx->input->stream_errno != 0) { + i_fatal("read(%s) failed: %s", ctx->socket_path, + i_stream_get_error(ctx->input)); + } else if (ctx->input->eof) { i_fatal("%s disconnected", ctx->socket_path); - else { + } else { i_fatal("read(%s) timed out (is director configured?)", ctx->socket_path); } @@ -79,62 +86,58 @@ static void director_connect(struct director_context *ctx) static void director_disconnect(struct director_context *ctx) { if (ctx->input != NULL) { - if (ctx->input->stream_errno != 0) - i_fatal("read(%s) failed: %m", ctx->socket_path); + if (ctx->input->stream_errno != 0) { + i_fatal("read(%s) failed: %s", ctx->socket_path, + i_stream_get_error(ctx->input)); + } i_stream_destroy(&ctx->input); } } static struct director_context * -cmd_director_init(int argc, char *argv[], const char *getopt_args, - doveadm_command_t *cmd) +cmd_director_init(struct doveadm_cmd_context *cctx) { struct director_context *ctx; - int c; - ctx = t_new(struct director_context, 1); - ctx->socket_path = t_strconcat(doveadm_settings->base_dir, - "/director-admin", NULL); - - while ((c = getopt(argc, argv, getopt_args)) > 0) { - switch (c) { - case 'a': - ctx->socket_path = optarg; - ctx->explicit_socket_path = TRUE; - break; - case 'f': - ctx->users_path = optarg; - break; - case 'F': - ctx->force_flush = TRUE; - break; - case 'h': - ctx->hash_map = TRUE; - break; - case 'u': - ctx->user_map = TRUE; - break; - case 't': - ctx->tag = optarg; - break; - default: - director_cmd_help(cmd); - } - } + if (!doveadm_cmd_param_str(cctx, "socket-path", &(ctx->socket_path))) + ctx->socket_path = t_strconcat(doveadm_settings->base_dir, + "/director-admin", NULL); + else + ctx->explicit_socket_path = TRUE; + if (!doveadm_cmd_param_bool(cctx, "user-map", &(ctx->user_map))) + ctx->user_map = FALSE; + if (!doveadm_cmd_param_bool(cctx, "hash-map", &(ctx->hash_map))) + ctx->hash_map = FALSE; + if (!doveadm_cmd_param_bool(cctx, "force-flush", &(ctx->force_flush))) + ctx->force_flush = FALSE; + if (!doveadm_cmd_param_istream(cctx, "users-file", &(ctx->users_input))) + ctx->users_input = NULL; + if (!doveadm_cmd_param_str(cctx, "tag", &(ctx->tag))) + ctx->tag = NULL; + if (!doveadm_cmd_param_str(cctx, "user", &(ctx->user))) + ctx->user = NULL; + if (!doveadm_cmd_param_str(cctx, "host", &(ctx->host))) + ctx->host = NULL; + if (!doveadm_cmd_param_str(cctx, "ip", &(ctx->ip))) + ctx->ip = NULL; + if (!doveadm_cmd_param_str(cctx, "port", &(ctx->port))) + ctx->port = NULL; + if (!doveadm_cmd_param_str(cctx, "vhost-count", &(ctx->vhost_count))) + ctx->vhost_count = NULL; if (!ctx->user_map) director_connect(ctx); return ctx; } + static void -cmd_director_status_user(struct director_context *ctx, char *argv[]) +cmd_director_status_user(struct director_context *ctx) { - const char *user = argv[0], *tag = argv[1]; const char *line, *const *args; unsigned int expires; - director_send(ctx, t_strdup_printf("USER-LOOKUP\t%s\t%s\n", user, - tag != NULL ? tag : "")); + director_send(ctx, t_strdup_printf("USER-LOOKUP\t%s\t%s\n", ctx->user, + ctx->tag != NULL ? ctx->tag : "")); line = i_stream_read_next_line(ctx->input); if (line == NULL) { i_error("Lookup failed"); @@ -150,25 +153,38 @@ cmd_director_status_user(struct director_context *ctx, char *argv[]) return; } + doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED); + + doveadm_print_header_simple("status"); + doveadm_print_header_simple("expires"); + doveadm_print_header_simple("hashed"); + doveadm_print_header_simple("initial-config"); + + doveadm_print_formatted_set_format("Current: %{status} (expires %{expires})\n" \ + "Hashed: %{hashed}\n" \ + "Initial config: %{initial-config}\n"); + if (args[0][0] != '\0') { - printf("Current: %s (expires %s)\n", - args[0], unixdate2str(expires)); + doveadm_print(args[0]); + doveadm_print(unixdate2str(expires)); } else { - printf("Current: not assigned\n"); + doveadm_print("n/a"); + doveadm_print("-1"); } - printf("Hashed: %s\n", args[2]); - printf("Initial config: %s\n", args[3]); + doveadm_print(args[2]); + doveadm_print(args[3]); + director_disconnect(ctx); } -static void cmd_director_status(int argc, char *argv[]) +static void cmd_director_status(struct doveadm_cmd_context *cctx) { struct director_context *ctx; const char *line, *const *args; - ctx = cmd_director_init(argc, argv, "a:t:", cmd_director_status); - if (argv[optind] != NULL) { - cmd_director_status_user(ctx, argv+optind); + ctx = cmd_director_init(cctx); + if (ctx->user != NULL) { + cmd_director_status_user(ctx); return; } @@ -254,27 +270,20 @@ userdb_get_user_list(const char *auth_socket_path, pool_t pool, } static void -user_file_get_user_list(const char *path, pool_t pool, +user_file_get_user_list(struct istream *input, pool_t pool, HASH_TABLE_TYPE(user_list) users) { - struct istream *input; const char *username; - int fd; - fd = open(path, O_RDONLY); - if (fd == -1) - i_fatal("open(%s) failed: %m", path); - input = i_stream_create_fd_autoclose(&fd, (size_t)-1); while ((username = i_stream_read_next_line(input)) != NULL) user_list_add(username, pool, users); - i_stream_unref(&input); } -static void director_get_host(const char *host, struct ip_addr **ips_r, +static int director_get_host(const char *host, struct ip_addr **ips_r, unsigned int *ips_count_r) { struct ip_addr ip; - int ret; + int ret = 0; if (net_addr2ip(host, &ip) == 0) { *ips_r = t_new(struct ip_addr, 1); @@ -283,10 +292,14 @@ static void director_get_host(const char *host, struct ip_addr **ips_r, } else { ret = net_gethostbyname(host, ips_r, ips_count_r); if (ret != 0) { - i_fatal("gethostname(%s) failed: %s", host, + i_error("gethostname(%s) failed: %s", host, net_gethosterror(ret)); + doveadm_exit_code = EX_TEMPFAIL; + return ret; } } + + return ret; } static bool ip_find(const struct ip_addr *ips, unsigned int ips_count, @@ -301,7 +314,7 @@ static bool ip_find(const struct ip_addr *ips, unsigned int ips_count, return FALSE; } -static void cmd_director_map(int argc, char *argv[]) +static void cmd_director_map(struct doveadm_cmd_context *cctx) { struct director_context *ctx; const char *line, *const *args; @@ -311,17 +324,16 @@ static void cmd_director_map(int argc, char *argv[]) struct user_list *user; unsigned int ips_count, user_hash, expires; - ctx = cmd_director_init(argc, argv, "a:f:hu", cmd_director_map); - argc -= optind; - argv += optind; - if (argc > 1 || - (ctx->hash_map && ctx->user_map) || - ((ctx->hash_map || ctx->user_map) && argc == 0)) - director_cmd_help(cmd_director_map); + ctx = cmd_director_init(cctx); + + if ((ctx->hash_map && ctx->user_map) && ctx->host == NULL) { + director_cmd_help(cctx->cmd); + return; + } if (ctx->user_map) { /* user -> hash mapping */ - user_hash = mail_user_hash(argv[0], doveadm_settings->director_username_hash); + user_hash = mail_user_hash(ctx->host, doveadm_settings->director_username_hash); doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); doveadm_print_header("hash", "hash", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); doveadm_print(t_strdup_printf("%u", user_hash)); @@ -329,22 +341,22 @@ static void cmd_director_map(int argc, char *argv[]) return; } - if (argv[0] == NULL || ctx->hash_map) + if (ctx->host == NULL || ctx->hash_map) ips_count = 0; else - director_get_host(argv[0], &ips, &ips_count); + if (director_get_host(ctx->host, &ips, &ips_count) != 0) return; pool = pool_alloconly_create("director map users", 1024*128); hash_table_create_direct(&users, pool, 0); - if (ctx->users_path == NULL) + if (ctx->users_input == NULL) userdb_get_user_list(NULL, pool, users); else - user_file_get_user_list(ctx->users_path, pool, users); + user_file_get_user_list(ctx->users_input, pool, users); if (ctx->hash_map) { /* hash -> usernames mapping */ - if (str_to_uint(argv[0], &user_hash) < 0) - i_fatal("Invalid username hash: %s", argv[0]); + if (str_to_uint(ctx->host, &user_hash) < 0) + i_fatal("Invalid username hash: %s", ctx->host); doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); doveadm_print_header("user", "user", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); @@ -407,37 +419,39 @@ static void cmd_director_map(int argc, char *argv[]) } static void -cmd_director_add_or_update(int argc, char *argv[], doveadm_command_t *cmd_func, - bool update) +cmd_director_add_or_update(struct doveadm_cmd_context *cctx, bool update) { const char *director_cmd = update ? "HOST-UPDATE" : "HOST-SET"; struct director_context *ctx; struct ip_addr *ips; unsigned int i, ips_count, vhost_count = UINT_MAX; - const char *host, *line; + const char *line, *host; string_t *cmd; - ctx = cmd_director_init(argc, argv, update ? "a:" : "a:t:", cmd_func); + ctx = cmd_director_init(cctx); if (ctx->tag != NULL && ctx->tag[0] == '\0') ctx->tag = NULL; - host = argv[optind++]; - if (host == NULL) - director_cmd_help(cmd_func); - if (argv[optind] != NULL) { - if (str_to_uint(argv[optind++], &vhost_count) < 0) - director_cmd_help(cmd_func); - } else if (strcmp(director_cmd, "HOST-UPDATE") == 0) - director_cmd_help(cmd_func); - - if (argv[optind] != NULL) - director_cmd_help(cmd_func); + if (ctx->host == NULL) { + director_cmd_help(cctx->cmd); + return; + } + if (ctx->vhost_count != NULL) { + if (str_to_uint(ctx->vhost_count, &vhost_count) < 0) { + director_cmd_help(cctx->cmd); + return; + } + } else if (update) { + director_cmd_help(cctx->cmd); + return; + } + host = ctx->host; if (ctx->tag == NULL) { - ctx->tag = strchr(host, '@'); + ctx->tag = strchr(ctx->host, '@'); if (ctx->tag != NULL) - host = t_strdup_until(host, ctx->tag++); + host = t_strdup_until(ctx->host, ctx->tag++); } - director_get_host(host, &ips, &ips_count); + if (director_get_host(host, &ips, &ips_count) != 0) return; cmd = t_str_new(128); for (i = 0; i < ips_count; i++) { str_truncate(cmd, 0); @@ -452,43 +466,45 @@ cmd_director_add_or_update(int argc, char *argv[], doveadm_command_t *cmd_func, for (i = 0; i < ips_count; i++) { line = i_stream_read_next_line(ctx->input); if (line == NULL || strcmp(line, "OK") != 0) { - fprintf(stderr, "%s: %s\n", net_ip2addr(&ips[i]), + i_error("%s: %s\n", net_ip2addr(&ips[i]), line == NULL ? "failed" : strcmp(line, "NOTFOUND") == 0 ? "doesn't exist" : line); doveadm_exit_code = EX_TEMPFAIL; } else if (doveadm_verbose) { - printf("%s: OK\n", net_ip2addr(&ips[i])); + i_info("%s: OK\n", net_ip2addr(&ips[i])); } } director_disconnect(ctx); } -static void cmd_director_add(int argc, char *argv[]) +static void cmd_director_add(struct doveadm_cmd_context *cctx) { - cmd_director_add_or_update(argc, argv, cmd_director_add, FALSE); + cmd_director_add_or_update(cctx, FALSE); } -static void cmd_director_update(int argc, char *argv[]) +static void cmd_director_update(struct doveadm_cmd_context *cctx) { - cmd_director_add_or_update(argc, argv, cmd_director_update, TRUE); + cmd_director_add_or_update(cctx, TRUE); } static void -cmd_director_ipcmd(const char *cmd_name, doveadm_command_t *cmd, - const char *success_result, int argc, char *argv[]) +cmd_director_ipcmd(const char *cmd_name, const char *success_result, + struct doveadm_cmd_context *cctx) { struct director_context *ctx; struct ip_addr *ips; unsigned int i, ips_count; const char *host, *line; - ctx = cmd_director_init(argc, argv, "a:", cmd); - host = argv[optind++]; - if (host == NULL || argv[optind] != NULL) - director_cmd_help(cmd); + ctx = cmd_director_init(cctx); + host = ctx->host; + if (host == NULL) { + director_cmd_help(cctx->cmd); + return; + } - director_get_host(host, &ips, &ips_count); + if (director_get_host(host, &ips, &ips_count) != 0) return; for (i = 0; i < ips_count; i++) { director_send(ctx, t_strdup_printf( "%s\t%s\n", cmd_name, net_ip2addr(&ips[i]))); @@ -496,55 +512,52 @@ cmd_director_ipcmd(const char *cmd_name, doveadm_command_t *cmd, for (i = 0; i < ips_count; i++) { line = i_stream_read_next_line(ctx->input); if (line != NULL && strcmp(line, "NOTFOUND") == 0) { - fprintf(stderr, "%s: doesn't exist\n", + i_error("%s: doesn't exist\n", net_ip2addr(&ips[i])); if (doveadm_exit_code == 0) doveadm_exit_code = DOVEADM_EX_NOTFOUND; } else if (line == NULL || strcmp(line, "OK") != 0) { - fprintf(stderr, "%s: %s\n", net_ip2addr(&ips[i]), + i_error("%s: %s\n", net_ip2addr(&ips[i]), line == NULL ? "failed" : line); doveadm_exit_code = EX_TEMPFAIL; } else if (doveadm_verbose) { - printf("%s: %s\n", net_ip2addr(&ips[i]), success_result); + i_info("%s: %s\n", net_ip2addr(&ips[i]), success_result); } } director_disconnect(ctx); } -static void cmd_director_remove(int argc, char *argv[]) +static void cmd_director_remove(struct doveadm_cmd_context *cctx) { - cmd_director_ipcmd("HOST-REMOVE", cmd_director_remove, - "removed", argc, argv); + cmd_director_ipcmd("HOST-REMOVE", "removed", cctx); } -static void cmd_director_up(int argc, char *argv[]) +static void cmd_director_up(struct doveadm_cmd_context *cctx) { - cmd_director_ipcmd("HOST-UP", cmd_director_up, - "up", argc, argv); + cmd_director_ipcmd("HOST-UP", "up", cctx); } -static void cmd_director_down(int argc, char *argv[]) +static void cmd_director_down(struct doveadm_cmd_context *cctx) { - cmd_director_ipcmd("HOST-DOWN", cmd_director_down, - "down", argc, argv); + cmd_director_ipcmd("HOST-DOWN", "down", cctx); } -static void cmd_director_move(int argc, char *argv[]) +static void cmd_director_move(struct doveadm_cmd_context *cctx) { struct director_context *ctx; struct ip_addr *ips; unsigned int ips_count, user_hash; - const char *host, *line, *ip_str; + const char *line, *ip_str; - ctx = cmd_director_init(argc, argv, "a:", cmd_director_move); - if (argv[optind] == NULL || argv[optind+1] == NULL || - argv[optind+2] != NULL) - director_cmd_help(cmd_director_move); + ctx = cmd_director_init(cctx); + if (ctx->user == NULL || ctx->host == NULL) { + director_cmd_help(cctx->cmd); + return; + } - user_hash = mail_user_hash(argv[optind++], doveadm_settings->director_username_hash); - host = argv[optind]; + user_hash = mail_user_hash(ctx->user, doveadm_settings->director_username_hash); - director_get_host(host, &ips, &ips_count); + if (director_get_host(ctx->host, &ips, &ips_count) != 0) return; ip_str = net_ip2addr(&ips[0]); director_send(ctx, t_strdup_printf( "USER-MOVE\t%u\t%s\n", user_hash, ip_str)); @@ -554,7 +567,7 @@ static void cmd_director_move(int argc, char *argv[]) doveadm_exit_code = EX_TEMPFAIL; } else if (strcmp(line, "OK") == 0) { if (doveadm_verbose) - printf("User hash %u moved to %s\n", user_hash, ip_str); + i_info("User hash %u moved to %s\n", user_hash, ip_str); } else if (strcmp(line, "NOTFOUND") == 0) { i_error("Host '%s' doesn't exist", ip_str); doveadm_exit_code = DOVEADM_EX_NOTFOUND; @@ -569,25 +582,25 @@ static void cmd_director_move(int argc, char *argv[]) director_disconnect(ctx); } -static void cmd_director_kick(int argc, char *argv[]) +static void cmd_director_kick(struct doveadm_cmd_context *cctx) { struct director_context *ctx; - const char *username, *line; - - ctx = cmd_director_init(argc, argv, "a:", cmd_director_kick); - if (argv[optind] == NULL || argv[optind+1] != NULL) - director_cmd_help(cmd_director_kick); + const char *line; - username = argv[optind]; + ctx = cmd_director_init(cctx); + if (ctx->user == NULL) { + director_cmd_help(cctx->cmd); + return; + } - director_send(ctx, t_strdup_printf("USER-KICK\t%s\n", username)); + director_send(ctx, t_strdup_printf("USER-KICK\t%s\n", ctx->user)); line = i_stream_read_next_line(ctx->input); if (line == NULL) { i_error("failed"); doveadm_exit_code = EX_TEMPFAIL; } else if (strcmp(line, "OK") == 0) { if (doveadm_verbose) - printf("User %s kicked\n", username); + i_info("User %s kicked", ctx->user); } else { i_error("failed: %s", line); doveadm_exit_code = EX_TEMPFAIL; @@ -610,37 +623,33 @@ static void cmd_director_flush_all(struct director_context *ctx) i_error("failed: %s", line); doveadm_exit_code = EX_TEMPFAIL; } else if (doveadm_verbose) - printf("flushed\n"); + i_info("flushed"); director_disconnect(ctx); } -static void cmd_director_flush(int argc, char *argv[]) +static void cmd_director_flush(struct doveadm_cmd_context *cctx) { struct director_context *ctx; struct ip_addr *ips; unsigned int i, ips_count; struct ip_addr ip; - const char *host, *line; - int ret; + const char *line; - ctx = cmd_director_init(argc, argv, "a:F", cmd_director_flush); - host = argv[optind++]; - if (host == NULL || argv[optind] != NULL) - director_cmd_help(cmd_director_flush); + ctx = cmd_director_init(cctx); + if (ctx->host == NULL) { + director_cmd_help(cctx->cmd); + return; + } - if (strcmp(host, "all") == 0) { + if (strcmp(ctx->host, "all") == 0) { cmd_director_flush_all(ctx); return; } - if (net_addr2ip(host, &ip) == 0) { + if (net_addr2ip(ctx->host, &ip) == 0) { ips = &ip; ips_count = 1; } else { - ret = net_gethostbyname(host, &ips, &ips_count); - if (ret != 0) { - i_fatal("gethostname(%s) failed: %s", host, - net_gethosterror(ret)); - } + if (director_get_host(ctx->host, &ips, &ips_count) != 0) return; } for (i = 0; i < ips_count; i++) { @@ -651,42 +660,38 @@ static void cmd_director_flush(int argc, char *argv[]) for (i = 0; i < ips_count; i++) { line = i_stream_read_next_line(ctx->input); if (line != NULL && strcmp(line, "NOTFOUND") == 0) { - fprintf(stderr, "%s: doesn't exist\n", + i_warning("%s: doesn't exist", net_ip2addr(&ips[i])); if (doveadm_exit_code == 0) doveadm_exit_code = DOVEADM_EX_NOTFOUND; } else if (line == NULL || strcmp(line, "OK") != 0) { - fprintf(stderr, "%s: %s\n", net_ip2addr(&ips[i]), + i_warning("%s: %s", net_ip2addr(&ips[i]), line == NULL ? "failed" : line); doveadm_exit_code = EX_TEMPFAIL; } else if (doveadm_verbose) { - printf("%s: flushed\n", net_ip2addr(&ips[i])); + i_info("%s: flushed", net_ip2addr(&ips[i])); } } director_disconnect(ctx); } -static void ATTR_FORMAT(3, 4) -director_dump_cmd(struct director_context *ctx, - const char *cmd, const char *args, ...) -{ - va_list va; - - va_start(va, args); - printf("doveadm director %s ", cmd); - if (ctx->explicit_socket_path) - printf("-a %s ", ctx->socket_path); - vprintf(args, va); - putchar('\n'); - va_end(va); -} - -static void cmd_director_dump(int argc, char *argv[]) +static void cmd_director_dump(struct doveadm_cmd_context *cctx) { struct director_context *ctx; const char *line, *const *args; - ctx = cmd_director_init(argc, argv, "a:", cmd_director_dump); + ctx = cmd_director_init(cctx); + + doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED); + if (ctx->explicit_socket_path) + doveadm_print_formatted_set_format("doveadm director %{command} -a %{socket-path} %{host} %{vhost_count}"); + else + doveadm_print_formatted_set_format("doveadm director %{command} %{host} %{vhost_count}"); + + doveadm_print_header_simple("command"); + doveadm_print_header_simple("socket-path"); + doveadm_print_header_simple("host"); + doveadm_print_header_simple("vhost_count"); director_send(ctx, "HOST-LIST\n"); while ((line = i_stream_read_next_line(ctx->input)) != NULL) { @@ -695,8 +700,10 @@ static void cmd_director_dump(int argc, char *argv[]) T_BEGIN { args = t_strsplit_tab(line); if (str_array_length(args) >= 2) { - director_dump_cmd(ctx, "add", "%s %s", - args[0], args[1]); + doveadm_print("add"); + doveadm_print(ctx->socket_path); + doveadm_print(args[0]); + doveadm_print(args[1]); } } T_END; } @@ -705,7 +712,10 @@ static void cmd_director_dump(int argc, char *argv[]) while ((line = i_stream_read_next_line(ctx->input)) != NULL) { if (*line == '\0') break; - director_dump_cmd(ctx, "remove", "%s", line); + doveadm_print("remove"); + doveadm_print(ctx->socket_path); + doveadm_print(line); + doveadm_print(""); } if (line == NULL) { i_error("Director disconnected unexpectedly"); @@ -732,18 +742,20 @@ static void director_read_ok_reply(struct director_context *ctx) } } -static void cmd_director_ring_add(int argc, char *argv[]) +static void cmd_director_ring_add(struct doveadm_cmd_context *cctx) { struct director_context *ctx; struct ip_addr ip; in_port_t port = 0; string_t *str = t_str_new(64); - ctx = cmd_director_init(argc, argv, "a:", cmd_director_ring_add); - if (argv[optind] == NULL || - net_addr2ip(argv[optind], &ip) < 0 || - (argv[optind+1] != NULL && net_str2port(argv[optind+1], &port) < 0)) - director_cmd_help(cmd_director_ring_add); + ctx = cmd_director_init(cctx); + if (ctx->ip == NULL || + net_addr2ip(ctx->ip, &ip) < 0 || + (ctx->port && net_str2port(ctx->port, &port) < 0)) { + director_cmd_help(cctx->cmd); + return; + } str_printfa(str, "DIRECTOR-ADD\t%s", net_ip2addr(&ip)); if (port != 0) @@ -754,18 +766,20 @@ static void cmd_director_ring_add(int argc, char *argv[]) director_disconnect(ctx); } -static void cmd_director_ring_remove(int argc, char *argv[]) +static void cmd_director_ring_remove(struct doveadm_cmd_context *cctx) { struct director_context *ctx; struct ip_addr ip; string_t *str = t_str_new(64); in_port_t port = 0; - ctx = cmd_director_init(argc, argv, "a:", cmd_director_ring_remove); - if (argv[optind] == NULL || - net_addr2ip(argv[optind], &ip) < 0 || - (argv[optind+1] != NULL && net_str2port(argv[optind+1], &port) < 0)) - director_cmd_help(cmd_director_ring_remove); + ctx = cmd_director_init(cctx); + if (ctx->ip == NULL || + net_addr2ip(ctx->ip, &ip) < 0 || + (ctx->port != NULL && net_str2port(ctx->port, &port) < 0)) { + director_cmd_help(cctx->cmd); + return; + } str_printfa(str, "DIRECTOR-REMOVE\t%s", net_ip2addr(&ip)); if (port != 0) @@ -776,13 +790,13 @@ static void cmd_director_ring_remove(int argc, char *argv[]) director_disconnect(ctx); } -static void cmd_director_ring_status(int argc, char *argv[]) +static void cmd_director_ring_status(struct doveadm_cmd_context *cctx) { struct director_context *ctx; const char *line, *const *args; unsigned long l; - ctx = cmd_director_init(argc, argv, "a:", cmd_director_ring_status); + ctx = cmd_director_init(cctx); doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); doveadm_print_header_simple("director ip"); @@ -817,46 +831,152 @@ static void cmd_director_ring_status(int argc, char *argv[]) director_disconnect(ctx); } -struct doveadm_cmd doveadm_cmd_director[] = { - { cmd_director_status, "director status", - "[-a ] []" }, - { cmd_director_map, "director map", - "[-a ] [-f ] [-h | -u] []" }, - { cmd_director_add, "director add", - "[-a ] [-t ] []" }, - { cmd_director_update, "director update", - "[-a ] " }, - { cmd_director_up, "director up", - "[-a ] " }, - { cmd_director_down, "director down", - "[-a ] " }, - { cmd_director_remove, "director remove", - "[-a ] " }, - { cmd_director_move, "director move", - "[-a ] " }, - { cmd_director_kick, "director kick", - "[-a ] " }, - { cmd_director_flush, "director flush", - "[-a ] [-f] |all" }, - { cmd_director_dump, "director dump", - "[-a ]" }, - { cmd_director_ring_add, "director ring add", - "[-a ] []" }, - { cmd_director_ring_remove, "director ring remove", - "[-a ] []" }, - { cmd_director_ring_status, "director ring status", - "[-a ]" } +struct doveadm_cmd_ver2 doveadm_cmd_director[] = { +{ + .name = "director status", + .cmd = cmd_director_status, + .usage = "[-a ] [] []", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "tag", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director map", + .cmd = cmd_director_map, + .usage = "[-a ] [-f ] [-h | -u] []", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('f', "users-file", CMD_PARAM_ISTREAM, 0) +DOVEADM_CMD_PARAM('h', "hash-map", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('u', "user-map", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director add", + .cmd = cmd_director_add, + .usage = "[-a ] [-t ] []", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('t', "tag", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "vhost-count", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director update", + .cmd = cmd_director_update, + .usage = "[-a ] ", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "vhost-count", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director up", + .cmd = cmd_director_up, + .usage = "[-a ] ", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director down", + .cmd = cmd_director_down, + .usage = "[-a ] ", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director remove", + .cmd = cmd_director_remove, + .usage = "[-a ] ", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director move", + .cmd = cmd_director_move, + .usage = "[-a ] ", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director kick", + .cmd = cmd_director_kick, + .usage = "[-a ] ", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director flush", + .cmd = cmd_director_flush, + .usage = "[-a ] [-F] |all", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('F', "force-flush", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director dump", + .cmd = cmd_director_dump, + .usage = "[-a ]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director ring add", + .cmd = cmd_director_ring_add, + .usage = "[-a ] []", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "ip", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "port", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director ring remove", + .cmd = cmd_director_ring_remove, + .usage = "[-a ] []", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "ip", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "port", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director ring status", + .cmd = cmd_director_ring_status, + .usage = "[-a ]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAMS_END +} }; -static void director_cmd_help(doveadm_command_t *cmd) +static void director_cmd_help(const struct doveadm_cmd_ver2 *cmd) { unsigned int i; for (i = 0; i < N_ELEMENTS(doveadm_cmd_director); i++) { - if (doveadm_cmd_director[i].cmd == cmd) - help(&doveadm_cmd_director[i]); + if (doveadm_cmd_director+i == cmd) + help_ver2(&doveadm_cmd_director[i]); } - i_unreached(); } void doveadm_register_director_commands(void) @@ -864,5 +984,5 @@ void doveadm_register_director_commands(void) unsigned int i; for (i = 0; i < N_ELEMENTS(doveadm_cmd_director); i++) - doveadm_register_cmd(&doveadm_cmd_director[i]); + doveadm_cmd_register_ver2(&doveadm_cmd_director[i]); } diff --git a/src/doveadm/doveadm-dump-index.c b/src/doveadm/doveadm-dump-index.c index 9edffe754ef..3e048444c4a 100644 --- a/src/doveadm/doveadm-dump-index.c +++ b/src/doveadm/doveadm-dump-index.c @@ -77,6 +77,17 @@ struct mailbox_list_index_record { guid_128_t guid; uint32_t uid_validity; }; +struct mailbox_list_index_msgs_record { + uint32_t messages; + uint32_t unseen; + uint32_t recent; + uint32_t uidnext; +}; +struct mailbox_index_vsize { + uint64_t vsize; + uint32_t highest_uid; + uint32_t message_count; +}; struct fts_index_header { uint32_t last_indexed_uid; @@ -610,7 +621,7 @@ static void dump_record(struct mail_index_view *view, unsigned int seq) } else if (strcmp(ext[i].name, "obox") == 0) { const struct obox_mail_index_record *orec = data; printf(" : guid = %s\n", guid_128_to_string(orec->guid)); - printf(" : oid = %s\n", guid_128_to_string(orec->oid)); + printf(" : oid = %s\n", binary_to_hex(orec->oid, ext[i].record_size - sizeof(orec->guid))); } else if (strcmp(ext[i].name, "mobox") == 0) { const struct mobox_mail_index_record *orec = data; printf(" : map_uid = %u\n", orec->map_uid); @@ -626,6 +637,18 @@ static void dump_record(struct mail_index_view *view, unsigned int seq) printf(" : parent_uid = %u\n", lrec->parent_uid); printf(" : guid = %s\n", guid_128_to_string(lrec->guid)); printf(" : uid_validity = %u\n", lrec->uid_validity); + } else if (strcmp(ext[i].name, "msgs") == 0) { + const struct mailbox_list_index_msgs_record *lrec = data; + printf(" : messages = %u\n", lrec->messages); + printf(" : unseen = %u\n", lrec->unseen); + printf(" : recent = %u\n", lrec->recent); + printf(" : uidnext = %u\n", lrec->uidnext); + } else if (strcmp(ext[i].name, "vsize") == 0) { + const struct mailbox_index_vsize *vrec = data; + printf(" : vsize = %llu\n", + (unsigned long long)vrec->vsize); + printf(" : highest_uid = %u\n", vrec->highest_uid); + printf(" : message_count = %u\n", vrec->message_count); } } } diff --git a/src/doveadm/doveadm-fs.c b/src/doveadm/doveadm-fs.c index fec6da90771..e785844d992 100644 --- a/src/doveadm/doveadm-fs.c +++ b/src/doveadm/doveadm-fs.c @@ -398,16 +398,12 @@ cmd_fs_delete_dir_recursive(struct fs *fs, unsigned int async_count, doveadm_fs_delete_async_finish(&ctx); } -static void -cmd_fs_delete_recursive(int argc, char *argv[], unsigned int async_count) +static void cmd_fs_delete_recursive_path(struct fs *fs, const char *path, + unsigned int async_count) { - struct fs *fs; struct fs_file *file; - const char *path; unsigned int path_len; - fs = cmd_fs_init(&argc, &argv, 1, cmd_fs_delete); - path = argv[0]; path_len = strlen(path); if (path_len > 0 && path[path_len-1] != '/') path = t_strconcat(path, "/", NULL); @@ -423,6 +419,17 @@ cmd_fs_delete_recursive(int argc, char *argv[], unsigned int async_count) } fs_file_deinit(&file); } +} + +static void +cmd_fs_delete_recursive(int argc, char *argv[], unsigned int async_count) +{ + struct fs *fs; + unsigned int i; + + fs = cmd_fs_init(&argc, &argv, 0, cmd_fs_delete); + for (i = 0; argv[i] != NULL; i++) + cmd_fs_delete_recursive_path(fs, argv[i], async_count); fs_deinit(&fs); } diff --git a/src/doveadm/doveadm-mail-fetch.c b/src/doveadm/doveadm-mail-fetch.c index d46aaa6910e..2fd58976bdb 100644 --- a/src/doveadm/doveadm-mail-fetch.c +++ b/src/doveadm/doveadm-mail-fetch.c @@ -15,6 +15,7 @@ #include "mail-storage.h" #include "mail-search.h" #include "mail-namespace.h" +#include "imap-msgpart.h" #include "doveadm-print.h" #include "doveadm-mail.h" #include "doveadm-mailbox-list-iter.h" @@ -32,6 +33,9 @@ struct fetch_cmd_context { enum mail_fetch_field wanted_fields; const struct fetch_field *cur_field; + /* if print() returns -1, log this error if non-NULL. otherwise log + the storage error. */ + const char *print_error; }; struct fetch_field { @@ -106,6 +110,13 @@ static int fetch_modseq(struct fetch_cmd_context *ctx) return 0; } +static void +fetch_set_istream_error(struct fetch_cmd_context *ctx, struct istream *input) +{ + ctx->print_error = t_strdup_printf("read(%s) failed: %s", + i_stream_get_name(input), i_stream_get_error(input)); +} + static int fetch_hdr(struct fetch_cmd_context *ctx) { struct istream *input; @@ -116,7 +127,8 @@ static int fetch_hdr(struct fetch_cmd_context *ctx) return -1; input = i_stream_create_limit(input, hdr_size.physical_size); - ret = doveadm_print_istream(input); + if ((ret = doveadm_print_istream(input) < 0)) + fetch_set_istream_error(ctx, input); i_stream_unref(&input); return ret; } @@ -186,16 +198,45 @@ static int fetch_hdr_field(struct fetch_cmd_context *ctx) return 0; } +static int fetch_body_field(struct fetch_cmd_context *ctx) +{ + const char *name = ctx->cur_field->name; + struct imap_msgpart *msgpart; + struct imap_msgpart_open_result result; + bool binary; + int ret; + + binary = strncmp(name, "binary.", 7) == 0; + name += binary ? 7 : 5; + if (imap_msgpart_parse(name, &msgpart) < 0) + i_unreached(); /* we already verified this was ok */ + if (binary) + imap_msgpart_set_decode_to_binary(msgpart); + + if (imap_msgpart_open(ctx->mail, msgpart, &result) < 0) { + imap_msgpart_free(&msgpart); + return -1; + } + if ((ret = doveadm_print_istream(result.input) < 0)) + fetch_set_istream_error(ctx, result.input); + i_stream_unref(&result.input); + imap_msgpart_free(&msgpart); + return ret; +} + static int fetch_body(struct fetch_cmd_context *ctx) { struct istream *input; struct message_size hdr_size; + int ret; if (mail_get_stream(ctx->mail, &hdr_size, NULL, &input) < 0) return -1; i_stream_skip(input, hdr_size.physical_size); - return doveadm_print_istream(input); + if ((ret = doveadm_print_istream(input) < 0)) + fetch_set_istream_error(ctx, input); + return ret; } static int fetch_body_snippet(struct fetch_cmd_context *ctx) @@ -213,10 +254,13 @@ static int fetch_body_snippet(struct fetch_cmd_context *ctx) static int fetch_text(struct fetch_cmd_context *ctx) { struct istream *input; + int ret; if (mail_get_stream(ctx->mail, NULL, NULL, &input) < 0) return -1; - return doveadm_print_istream(input); + if ((ret = doveadm_print_istream(input) < 0)) + fetch_set_istream_error(ctx, input); + return ret; } static int fetch_text_utf8(struct fetch_cmd_context *ctx) @@ -476,7 +520,7 @@ static void print_fetch_fields(void) { unsigned int i; - fprintf(stderr, "Available fetch fields: %s", fetch_fields[0].name); + fprintf(stderr, "Available fetch fields: hdr. body.
binary.
%s", fetch_fields[0].name); for (i = 1; i < N_ELEMENTS(fetch_fields); i++) fprintf(stderr, " %s", fetch_fields[i].name); fprintf(stderr, "\n"); @@ -486,11 +530,15 @@ static void parse_fetch_fields(struct fetch_cmd_context *ctx, const char *str) { const char *const *fields, *name; const struct fetch_field *field; - struct fetch_field hdr_field; + struct fetch_field hdr_field, body_field; + struct imap_msgpart *msgpart; memset(&hdr_field, 0, sizeof(hdr_field)); hdr_field.print = fetch_hdr_field; + memset(&body_field, 0, sizeof(body_field)); + body_field.print = fetch_body_field; + t_array_init(&ctx->fields, 32); t_array_init(&ctx->header_fields, 32); fields = t_strsplit_spaces(str, " "); @@ -504,6 +552,19 @@ static void parse_fetch_fields(struct fetch_cmd_context *ctx, const char *str) array_append(&ctx->fields, &hdr_field, 1); name = t_strcut(name, '.'); array_append(&ctx->header_fields, &name, 1); + } else if (strncmp(name, "body.", 5) == 0 || + strncmp(name, "binary.", 7) == 0) { + bool binary = strncmp(name, "binary.", 7) == 0; + body_field.name = t_strarray_join(t_strsplit(name, ","), " "); + + name += binary ? 7 : 5; + if (imap_msgpart_parse(name, &msgpart) < 0) { + print_fetch_fields(); + i_fatal("Unknown fetch section: %s", name); + } + array_append(&ctx->fields, &body_field, 1); + ctx->wanted_fields |= imap_msgpart_get_fetch_data(msgpart); + imap_msgpart_free(&msgpart); } else { field = fetch_field_find(name); if (field == NULL) { @@ -528,8 +589,11 @@ static int cmd_fetch_mail(struct fetch_cmd_context *ctx) if (field->print(ctx) < 0) { i_error("fetch(%s) failed for box=%s uid=%u: %s", field->name, mailbox_get_vname(mail->box), - mail->uid, mailbox_get_last_error(mail->box, NULL)); + mail->uid, + ctx->print_error != NULL ? ctx->print_error : + mailbox_get_last_error(mail->box, NULL)); doveadm_mail_failed_mailbox(&ctx->ctx, mail->box); + ctx->print_error = NULL; ret = -1; } } diff --git a/src/doveadm/doveadm-mail-index.c b/src/doveadm/doveadm-mail-index.c index 1667e122703..b0e1207a819 100644 --- a/src/doveadm/doveadm-mail-index.c +++ b/src/doveadm/doveadm-mail-index.c @@ -281,7 +281,7 @@ struct doveadm_cmd_ver2 doveadm_cmd_index_ver2 = { DOVEADM_CMD_PARAMS_START DOVEADM_CMD_MAIL_COMMON DOVEADM_CMD_PARAM('q',"queue",CMD_PARAM_BOOL,0) -DOVEADM_CMD_PARAM('n',"max-recent",CMD_PARAM_INT64,0) +DOVEADM_CMD_PARAM('n',"max-recent",CMD_PARAM_STR,0) DOVEADM_CMD_PARAM('\0',"mailbox-mask",CMD_PARAM_STR,CMD_PARAM_FLAG_POSITIONAL) DOVEADM_CMD_PARAMS_END }; diff --git a/src/doveadm/doveadm-mail-mailbox.c b/src/doveadm/doveadm-mail-mailbox.c index 7f16fed061d..a300e844b65 100644 --- a/src/doveadm/doveadm-mail-mailbox.c +++ b/src/doveadm/doveadm-mail-mailbox.c @@ -34,6 +34,8 @@ struct delete_cmd_context { struct doveadm_mailbox_cmd_context ctx; ARRAY_TYPE(const_string) mailboxes; bool recursive; + bool require_empty; + bool unsafe; }; struct rename_cmd_context { @@ -47,6 +49,12 @@ struct list_cmd_context { bool mutf7; }; +struct update_cmd_context { + struct doveadm_mailbox_cmd_context ctx; + const char *mailbox; + struct mailbox_update update; +}; + void doveadm_mailbox_args_check(const char *const args[]) { unsigned int i; @@ -328,8 +336,11 @@ cmd_mailbox_delete_run(struct doveadm_mail_cmd_context *_ctx, const char *const *namep; ARRAY_TYPE(const_string) recursive_mailboxes; const ARRAY_TYPE(const_string) *mailboxes = &ctx->mailboxes; - int ret = 0; + enum mailbox_flags mailbox_flags = 0; + int ret = 0, ret2; + if (ctx->unsafe) + mailbox_flags |= MAILBOX_FLAG_DELETE_UNSAFE; if (ctx->recursive) { t_array_init(&recursive_mailboxes, 32); array_foreach(&ctx->mailboxes, namep) { @@ -349,9 +360,11 @@ cmd_mailbox_delete_run(struct doveadm_mail_cmd_context *_ctx, const char *name = *namep; ns = mail_namespace_find(user->namespaces, name); - box = mailbox_alloc(ns->list, name, 0); + box = mailbox_alloc(ns->list, name, mailbox_flags); storage = mailbox_get_storage(box); - if (mailbox_delete(box) < 0) { + ret2 = ctx->require_empty ? mailbox_delete_empty(box) : + mailbox_delete(box); + if (ret2 < 0) { i_error("Can't delete mailbox %s: %s", name, mailbox_get_last_error(box, NULL)); doveadm_mail_failed_mailbox(_ctx, box); @@ -400,6 +413,12 @@ cmd_mailbox_delete_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) case 's': ctx->ctx.subscriptions = TRUE; break; + case 'e': + ctx->require_empty = TRUE; + break; + case 'Z': + ctx->unsafe = TRUE; + break; default: return FALSE; } @@ -414,7 +433,7 @@ static struct doveadm_mail_cmd_context *cmd_mailbox_delete_alloc(void) ctx->ctx.ctx.v.init = cmd_mailbox_delete_init; ctx->ctx.ctx.v.run = cmd_mailbox_delete_run; ctx->ctx.ctx.v.parse_arg = cmd_mailbox_delete_parse_arg; - ctx->ctx.ctx.getopt_args = "rs"; + ctx->ctx.ctx.getopt_args = "ersZ"; p_array_init(&ctx->mailboxes, ctx->ctx.ctx.pool, 16); return &ctx->ctx.ctx; } @@ -556,6 +575,99 @@ static struct doveadm_mail_cmd_context *cmd_mailbox_unsubscribe_alloc(void) return cmd_mailbox_subscriptions_alloc(FALSE); } +static +void cmd_mailbox_update_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct update_cmd_context *ctx = (struct update_cmd_context *)_ctx; + + if (str_array_length(args) != 1) + doveadm_mail_help_name("mailbox update"); + + doveadm_mailbox_args_check(args); + + ctx->mailbox = args[0]; + + if ((ctx->update.min_first_recent_uid != 0 || + ctx->update.min_next_uid != 0) && + ctx->update.min_first_recent_uid > ctx->update.min_next_uid) { + i_fatal_status(EX_DATAERR, + "min_first_recent_uid > min_next_uid"); + } +} + +static +bool cmd_mailbox_update_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct update_cmd_context *ctx = (struct update_cmd_context *)_ctx; + + switch (c) { + case 'g': + if (guid_128_from_string(optarg, ctx->update.mailbox_guid) < 0) + doveadm_mail_help_name("mailbox update"); + break; + case 'V': + if (str_to_uint32(optarg, &(ctx->update.uid_validity)) < 0) + doveadm_mail_help_name("mailbox update"); + break; + case 'N': + if (str_to_uint32(optarg, &(ctx->update.min_next_uid)) < 0) + doveadm_mail_help_name("mailbox update"); + break; + case 'R': + if (str_to_uint32(optarg, &(ctx->update.min_first_recent_uid)) < 0) + doveadm_mail_help_name("mailbox update"); + break; + case 'H': + if (str_to_uint64(optarg, &(ctx->update.min_highest_modseq)) < 0) + doveadm_mail_help_name("mailbox update"); + break; + case 'P': + if (str_to_uint64(optarg, &(ctx->update.min_highest_pvt_modseq)) < 0) + doveadm_mail_help_name("mailbox update"); + break; + default: + return FALSE; + } + return TRUE; +} + +static +int cmd_mailbox_update_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct update_cmd_context *ctx = (struct update_cmd_context *)_ctx; + struct mail_namespace *ns; + struct mailbox *box; + enum mail_error mail_error; + int ret = 0; + + ns = mail_namespace_find(user->namespaces, ctx->mailbox); + box = mailbox_alloc(ns->list, ctx->mailbox, 0); + + if ((ret = mailbox_update(box, &(ctx->update))) != 0) { + i_error("Cannot update %s: %s", + ctx->mailbox, + mailbox_get_last_error(box, &mail_error)); + doveadm_mail_failed_error(_ctx, mail_error); + } + + mailbox_free(&box); + + return ret; +} + +static +struct doveadm_mail_cmd_context *cmd_mailbox_update_alloc(void) +{ + struct update_cmd_context *ctx; + ctx = doveadm_mail_cmd_alloc(struct update_cmd_context); + ctx->ctx.ctx.v.parse_arg = cmd_mailbox_update_parse_arg; + ctx->ctx.ctx.v.init = cmd_mailbox_update_init; + ctx->ctx.ctx.v.run = cmd_mailbox_update_run; + return &ctx->ctx.ctx; +} + struct doveadm_cmd_ver2 doveadm_cmd_mailbox_list_ver2 = { .name = "mailbox list", .mail_cmd = cmd_mailbox_list_alloc, @@ -583,11 +695,14 @@ DOVEADM_CMD_PARAMS_END struct doveadm_cmd_ver2 doveadm_cmd_mailbox_delete_ver2 = { .name = "mailbox delete", - .mail_cmd = cmd_mailbox_delete_alloc, - .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-s] [...]", + .mail_cmd = cmd_mailbox_delete_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-e] [-r] [-s] [-Z] [...]", DOVEADM_CMD_PARAMS_START DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('e', "require-empty", CMD_PARAM_BOOL, 0) DOVEADM_CMD_PARAM('s', "subscriptions", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('r', "recursive", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('Z', "unsafe", CMD_PARAM_BOOL, 0) DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) DOVEADM_CMD_PARAMS_END }; @@ -623,3 +738,19 @@ DOVEADM_CMD_MAIL_COMMON DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) DOVEADM_CMD_PARAMS_END }; + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_update_ver2 = { + .name = "mailbox update", + .mail_cmd = cmd_mailbox_update_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[--mailbox-guid guid] [--uid-validity uid] [--min-next-uid uid] [--min-first-recent-uid uid] [--min-highest-modseq seq] [--min-highest-pvt-modseq seq] ", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('g', "mailbox-guid", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('V', "uid-validity", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('N', "min-next-uid", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('R', "min-first-recent-uid", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('H', "min-highest-modseq", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('P', "min-highest-pvt-modseq", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail.c b/src/doveadm/doveadm-mail.c index bdb12c13eb9..ca89a94c8cf 100644 --- a/src/doveadm/doveadm-mail.c +++ b/src/doveadm/doveadm-mail.c @@ -450,7 +450,8 @@ doveadm_mail_all_users(struct doveadm_mail_cmd_context *ctx, ctx->v.init(ctx, ctx->args); - mail_storage_service_all_init(ctx->storage_service); + mail_storage_service_all_init_mask(ctx->storage_service, + wildcard_user != NULL ? wildcard_user : ""); if (hook_doveadm_mail_init != NULL) hook_doveadm_mail_init(ctx); @@ -869,6 +870,7 @@ static struct doveadm_cmd_ver2 *mail_commands_ver2[] = { &doveadm_cmd_mailbox_rename_ver2, &doveadm_cmd_mailbox_subscribe_ver2, &doveadm_cmd_mailbox_unsubscribe_ver2, + &doveadm_cmd_mailbox_update_ver2, &doveadm_cmd_fetch_ver2, &doveadm_cmd_save_ver2, &doveadm_cmd_index_ver2, @@ -922,11 +924,9 @@ void doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx) { struct doveadm_mail_cmd_context *mctx; - const char *wildcard_user, *username_args[3] = { NULL, NULL, NULL }; + const char *wildcard_user; const char *fieldstr; - unsigned int username_args_count; - - ARRAY_TYPE(const_string) pargv; + ARRAY_TYPE(const_string) pargv, full_args; int i; struct doveadm_mail_cmd mail_cmd = { cctx->cmd->mail_cmd, cctx->cmd->name, cctx->cmd->usage @@ -942,6 +942,7 @@ doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx) mctx->cur_username = cctx->username; mctx->iterate_all_users = FALSE; wildcard_user = NULL; + p_array_init(&full_args, mctx->pool, 8); p_array_init(&pargv, mctx->pool, 8); for(i=0;iargc;i++) { @@ -952,7 +953,8 @@ doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx) if (strcmp(arg->name, "all-users") == 0) { mctx->iterate_all_users = arg->value.v_bool; - username_args[0] = "-A"; + fieldstr = "-A"; + array_append(&full_args, &fieldstr, 1); } else if (strcmp(arg->name, "socket-path") == 0) { doveadm_settings->doveadm_socket_path = arg->value.v_string; if (doveadm_settings->doveadm_worker_count == 0) @@ -960,8 +962,10 @@ doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx) } else if (strcmp(arg->name, "user") == 0) { mctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP; mctx->cur_username = arg->value.v_string; - username_args[0] = "-u"; - username_args[1] = arg->value.v_string; + + fieldstr = "-u"; + array_append(&full_args, &fieldstr, 1); + array_append(&full_args, &arg->value.v_string, 1); if (strchr(mctx->cur_username, '*') != NULL || strchr(mctx->cur_username, '?') != NULL) { wildcard_user = mctx->cur_username; @@ -973,8 +977,10 @@ doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx) mctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP; wildcard_user = "*"; mctx->users_list_input = arg->value.v_istream; - username_args[0] = "-F"; - username_args[1] = ""; /* value doesn't really matter */ + fieldstr = "-F"; + array_append(&full_args, &fieldstr, 1); + fieldstr = ""; /* value doesn't really matter */ + array_append(&full_args, &fieldstr, 1); i_stream_ref(mctx->users_list_input); } else if (strcmp(arg->name, "field") == 0 || strcmp(arg->name, "flag") == 0) { @@ -999,8 +1005,15 @@ doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx) /* Keep all named special parameters above this line */ } else if (mctx->v.parse_arg != NULL && arg->short_opt != '\0') { + const char *short_opt_str = p_strdup_printf( + mctx->pool, "-%c", arg->short_opt); + optarg = (char*)arg->value.v_string; mctx->v.parse_arg(mctx, arg->short_opt); + + array_append(&full_args, &short_opt_str, 1); + if (arg->type == CMD_PARAM_STR) + array_append(&full_args, &arg->value.v_string, 1); } else if ((arg->flags & CMD_PARAM_FLAG_POSITIONAL) != 0) { /* feed this into pargv */ if (arg->type == CMD_PARAM_ARRAY) @@ -1016,14 +1029,13 @@ doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx) } array_append_zero(&pargv); - /* -A, -u and -F parameters need to be included in full_args so that - they're sent to doveadm-server. This is needed so that - doveadm-server returns the username header when needed. */ - username_args_count = str_array_length(username_args); - if (username_args_count > 0) - array_insert(&pargv, 0, username_args, username_args_count); - mctx->args = array_idx(&pargv, username_args_count); - mctx->full_args = array_idx(&pargv, 0); + /* All the -parameters need to be included in full_args so that + they're sent to doveadm-server. */ + unsigned int args_pos = array_count(&full_args); + array_append_array(&full_args, &pargv); + + mctx->args = array_idx(&full_args, args_pos); + mctx->full_args = array_idx(&full_args, 0); mctx->cli = cctx->cli; doveadm_mail_cmd_exec(mctx, cctx, wildcard_user); diff --git a/src/doveadm/doveadm-mail.h b/src/doveadm/doveadm-mail.h index b33543899fa..fce8204fdcd 100644 --- a/src/doveadm/doveadm-mail.h +++ b/src/doveadm/doveadm-mail.h @@ -197,6 +197,7 @@ extern struct doveadm_cmd_ver2 doveadm_cmd_import_ver2; extern struct doveadm_cmd_ver2 doveadm_cmd_search_ver2; extern struct doveadm_cmd_ver2 doveadm_cmd_copy_ver2; extern struct doveadm_cmd_ver2 doveadm_cmd_move_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_update_ver2; #define DOVEADM_CMD_MAIL_COMMON \ DOVEADM_CMD_PARAM('A', "all-users", CMD_PARAM_BOOL, 0) \ diff --git a/src/doveadm/doveadm-penalty.c b/src/doveadm/doveadm-penalty.c index 36b9fd85f4b..2a4b33d8878 100644 --- a/src/doveadm/doveadm-penalty.c +++ b/src/doveadm/doveadm-penalty.c @@ -80,8 +80,10 @@ static void penalty_lookup(struct penalty_context *ctx) penalty_print_line(ctx, &penalty_line); } T_END; } - if (input->stream_errno != 0) - i_fatal("read(%s) failed: %m", ctx->anvil_path); + if (input->stream_errno != 0) { + i_fatal("read(%s) failed: %s", ctx->anvil_path, + i_stream_get_error(input)); + } i_stream_destroy(&input); } diff --git a/src/doveadm/doveadm-print-formatted.c b/src/doveadm/doveadm-print-formatted.c index 87e03c5f199..28347eb81d4 100644 --- a/src/doveadm/doveadm-print-formatted.c +++ b/src/doveadm/doveadm-print-formatted.c @@ -53,6 +53,9 @@ static void doveadm_print_formatted_flush(void) static void doveadm_print_formatted_print(const char *value) { + if (ctx.format == NULL) { + i_fatal("formatted formatter cannot be used without a format."); + } struct var_expand_table *entry = array_idx_modifiable(&ctx.headers, ctx.idx++); entry->value = value; diff --git a/src/doveadm/doveadm-replicator.c b/src/doveadm/doveadm-replicator.c index 6459a6c0f9f..390572b530f 100644 --- a/src/doveadm/doveadm-replicator.c +++ b/src/doveadm/doveadm-replicator.c @@ -48,9 +48,10 @@ static void replicator_connect(struct replicator_context *ctx) line = i_stream_read_next_line(ctx->input); alarm(0); if (line == NULL) { - if (ctx->input->stream_errno != 0) - i_fatal("read(%s) failed: %m", ctx->socket_path); - else if (ctx->input->eof) + if (ctx->input->stream_errno != 0) { + i_fatal("read(%s) failed: %s", ctx->socket_path, + i_stream_get_error(ctx->input)); + } else if (ctx->input->eof) i_fatal("%s disconnected", ctx->socket_path); else i_fatal("read(%s) timed out", ctx->socket_path); @@ -64,8 +65,10 @@ static void replicator_connect(struct replicator_context *ctx) static void replicator_disconnect(struct replicator_context *ctx) { - if (ctx->input->stream_errno != 0) - i_fatal("read(%s) failed: %m", ctx->socket_path); + if (ctx->input->stream_errno != 0) { + i_fatal("read(%s) failed: %s", ctx->socket_path, + i_stream_get_error(ctx->input)); + } i_stream_destroy(&ctx->input); } diff --git a/src/doveadm/doveadm-service.c b/src/doveadm/doveadm-service.c new file mode 100644 index 00000000000..95d97188e8c --- /dev/null +++ b/src/doveadm/doveadm-service.c @@ -0,0 +1,60 @@ +/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "write-full.h" +#include "istream.h" +#include "doveadm.h" + +#include + +static void cmd_service_stop(int argc, char *argv[]) +{ + const char *path, *line; + int fd; + + if (argc == 1) + help_ver2(&doveadm_cmd_service_stop_ver2); + + path = t_strconcat(doveadm_settings->base_dir, "/master", NULL); + fd = net_connect_unix(path); + if (fd == -1) + i_fatal("net_connect_unix(%s) failed: %m", path); + net_set_nonblock(fd, FALSE); + + string_t *cmd = t_str_new(128); + str_append(cmd, "VERSION\tmaster-client\t1\t0\nSTOP"); + for (int i = 1; i < argc; i++) { + str_append_c(cmd, '\t'); + str_append(cmd, argv[i]); + } + str_append_c(cmd, '\n'); + if (write_full(fd, str_data(cmd), str_len(cmd)) < 0) + i_error("write(%s) failed: %m", path); + + alarm(5); + struct istream *input = i_stream_create_fd(fd, IO_BLOCK_SIZE, FALSE); + if (i_stream_read_next_line(input) == NULL || + (line = i_stream_read_next_line(input)) == NULL) { + i_error("read(%s) failed: %s", path, i_stream_get_error(input)); + doveadm_exit_code = EX_TEMPFAIL; + } else if (line[0] == '-') { + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + i_error("%s", line+1); + } else if (line[0] != '+') { + i_error("Unexpected input from %s: %s", path, line); + doveadm_exit_code = EX_TEMPFAIL; + } + alarm(0); + i_stream_destroy(&input); + i_close_fd(&fd); +} + +struct doveadm_cmd_ver2 doveadm_cmd_service_stop_ver2 = { + .old_cmd = cmd_service_stop, + .name = "service stop", + .usage = " [ [...]]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "service", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-stats.c b/src/doveadm/doveadm-stats.c index 81e6f25623b..54201ed78a8 100644 --- a/src/doveadm/doveadm-stats.c +++ b/src/doveadm/doveadm-stats.c @@ -111,7 +111,7 @@ static void stats_dump(const char *path, const char *cmd) } while (args != NULL); } if (input->stream_errno != 0) - i_fatal("read(%s) failed: %m", path); + i_fatal("read(%s) failed: %s", path, i_stream_get_error(input)); i_stream_destroy(&input); } @@ -194,8 +194,10 @@ static void stats_read(struct top_context *ctx) hash_table_insert(ctx->sessions, line->id, line); } - if (ctx->input->stream_errno != 0) - i_fatal("read(%s) failed: %m", ctx->path); + if (ctx->input->stream_errno != 0) { + i_fatal("read(%s) failed: %s", ctx->path, + i_stream_get_error(ctx->input)); + } i_fatal("read(%s): unexpected EOF", ctx->path); } diff --git a/src/doveadm/doveadm-who.c b/src/doveadm/doveadm-who.c index 0fd7d94c2d0..d0f3b0f5d73 100644 --- a/src/doveadm/doveadm-who.c +++ b/src/doveadm/doveadm-who.c @@ -59,6 +59,8 @@ static int who_parse_line(const char *line, struct who_line *line_r) memset(line_r, 0, sizeof(*line_r)); + /* ident = service/ip/username (imap, pop3) + or service/username (lmtp) */ p = strchr(ident, '/'); if (p == NULL) return -1; @@ -66,12 +68,15 @@ static int who_parse_line(const char *line, struct who_line *line_r) return -1; line_r->service = t_strdup_until(ident, p++); line_r->username = strchr(p, '/'); - if (line_r->username == NULL) - return -1; + if (line_r->username == NULL) { + /* no IP */ + line_r->username = p; + } else { + ip_str = t_strdup_until(p, line_r->username++); + (void)net_addr2ip(ip_str, &line_r->ip); + } if (str_to_uint(refcount_str, &line_r->refcount) < 0) return -1; - ip_str = t_strdup_until(p, line_r->username++); - (void)net_addr2ip(ip_str, &line_r->ip); return 0; } @@ -164,8 +169,10 @@ void who_lookup(struct who_context *ctx, who_callback_t *callback) callback(ctx, &who_line); } T_END; } - if (input->stream_errno != 0) - i_fatal("read(%s) failed: %m", ctx->anvil_path); + if (input->stream_errno != 0) { + i_fatal("read(%s) failed: %s", ctx->anvil_path, + i_stream_get_error(input)); + } i_stream_destroy(&input); } diff --git a/src/doveadm/doveadm-zlib.c b/src/doveadm/doveadm-zlib.c index a459a46058a..fe6f19be71f 100644 --- a/src/doveadm/doveadm-zlib.c +++ b/src/doveadm/doveadm-zlib.c @@ -122,8 +122,8 @@ static void server_input(struct client *client) if (i_stream_read(client->input) == -1) { if (client->input->stream_errno != 0) { - errno = client->input->stream_errno; - i_fatal("read(server) failed: %m"); + i_fatal("read(server) failed: %s", + i_stream_get_error(client->input)); } i_info("Server disconnected"); diff --git a/src/doveadm/doveadm.c b/src/doveadm/doveadm.c index 677047e0083..8982b9c8cec 100644 --- a/src/doveadm/doveadm.c +++ b/src/doveadm/doveadm.c @@ -255,6 +255,7 @@ static void doveadm_read_settings(void) memset(&input, 0, sizeof(input)); input.roots = set_roots; input.module = "doveadm"; + input.service = "doveadm"; input.preserve_user = TRUE; input.preserve_home = TRUE; if (master_service_settings_read(master_service, &input, @@ -333,7 +334,7 @@ int main(int argc, char *argv[]) doveadm_cmds_init(); for (i = 0; i < N_ELEMENTS(doveadm_cmdline_commands); i++) doveadm_register_cmd(doveadm_cmdline_commands[i]); - + doveadm_register_auth_commands(); doveadm_cmd_register_ver2(&doveadm_cmd_stats_top_ver2); if (cmd_name != NULL && (quick_init || diff --git a/src/doveadm/dsync/dsync-brain-mailbox-tree.c b/src/doveadm/dsync/dsync-brain-mailbox-tree.c index 674cb0a4868..786efa50531 100644 --- a/src/doveadm/dsync/dsync-brain-mailbox-tree.c +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c @@ -310,7 +310,8 @@ static void dsync_brain_mailbox_trees_sync(struct dsync_brain *brain) break; } } - dsync_mailbox_trees_sync_deinit(&ctx); + if (dsync_mailbox_trees_sync_deinit(&ctx) < 0) + brain->failed = TRUE; } bool dsync_brain_recv_mailbox_tree(struct dsync_brain *brain) diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c index f71c7aa808e..bd380645bbf 100644 --- a/src/doveadm/dsync/dsync-brain-mailbox.c +++ b/src/doveadm/dsync/dsync-brain-mailbox.c @@ -343,10 +343,6 @@ void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain) i_assert(brain->box != NULL); - if (brain->require_full_resync) { - brain->mailbox_state.last_uidvalidity = 0; - brain->require_full_resync = FALSE; - } array_append(&brain->remote_mailbox_states, &brain->mailbox_state, 1); if (brain->box_exporter != NULL) { const char *errstr; @@ -360,7 +356,7 @@ void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain) if (brain->box_importer != NULL) { uint32_t last_common_uid, last_messages_count; uint64_t last_common_modseq, last_common_pvt_modseq; - bool changes_during_sync; + bool changes_during_sync, require_full_resync; i_assert(brain->failed); (void)dsync_mailbox_import_deinit(&brain->box_importer, @@ -370,7 +366,10 @@ void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain) &last_common_pvt_modseq, &last_messages_count, &changes_during_sync, + &require_full_resync, &brain->mail_error); + if (require_full_resync) + brain->require_full_resync = TRUE; } if (brain->log_scan != NULL) dsync_transaction_log_scan_deinit(&brain->log_scan); diff --git a/src/doveadm/dsync/dsync-brain-mails.c b/src/doveadm/dsync/dsync-brain-mails.c index a9ab009c5c9..5a934e4f662 100644 --- a/src/doveadm/dsync/dsync-brain-mails.c +++ b/src/doveadm/dsync/dsync-brain-mails.c @@ -219,6 +219,7 @@ static bool dsync_brain_send_mail_request(struct dsync_brain *brain) static void dsync_brain_sync_half_finished(struct dsync_brain *brain) { struct dsync_mailbox_state state; + bool require_full_resync; if (brain->box_recv_state < DSYNC_BOX_STATE_RECV_LAST_COMMON || brain->box_send_state < DSYNC_BOX_STATE_RECV_LAST_COMMON) @@ -246,13 +247,25 @@ static void dsync_brain_sync_half_finished(struct dsync_brain *brain) &state.last_common_pvt_modseq, &state.last_messages_count, &state.changes_during_sync, + &require_full_resync, &brain->mail_error) < 0) { - brain->failed = TRUE; - return; + if (require_full_resync) { + /* don't treat this as brain failure or the + state won't be sent to the other brain. + this also means we'll continue syncing the + following mailboxes. */ + brain->require_full_resync = TRUE; + } else { + brain->failed = TRUE; + } } if (state.changes_during_sync) brain->changes_during_sync = TRUE; } + if (brain->require_full_resync) { + state.last_uidvalidity = 0; + state.changes_during_sync = TRUE; + } brain->mailbox_state = state; dsync_ibc_send_mailbox_state(brain->ibc, &state); } diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c index 915d4f23f77..22b07aa6c93 100644 --- a/src/doveadm/dsync/dsync-brain.c +++ b/src/doveadm/dsync/dsync-brain.c @@ -568,16 +568,19 @@ static bool dsync_brain_finish(struct dsync_brain *brain) { const char *error; enum mail_error mail_error; + bool require_full_resync; enum dsync_ibc_recv_ret ret; if (!brain->master_brain) { dsync_ibc_send_finish(brain->ibc, brain->failed ? "dsync failed" : NULL, - brain->mail_error); + brain->mail_error, + brain->require_full_resync); brain->state = DSYNC_STATE_DONE; return TRUE; } - ret = dsync_ibc_recv_finish(brain->ibc, &error, &mail_error); + ret = dsync_ibc_recv_finish(brain->ibc, &error, &mail_error, + &require_full_resync); if (ret == DSYNC_IBC_RECV_RET_TRYAGAIN) return FALSE; if (error != NULL) { @@ -587,6 +590,8 @@ static bool dsync_brain_finish(struct dsync_brain *brain) (brain->mail_error == 0 || brain->mail_error == MAIL_ERROR_TEMP)) brain->mail_error = mail_error; } + if (require_full_resync) + brain->require_full_resync = TRUE; brain->state = DSYNC_STATE_DONE; return TRUE; } @@ -705,6 +710,9 @@ void dsync_brain_get_state(struct dsync_brain *brain, string_t *output) const uint8_t *guid_p; uint8_t *guid; + if (brain->require_full_resync) + return; + /* update mailbox states */ array_foreach(&brain->remote_mailbox_states, new_state) { guid_p = new_state->mailbox_guid; diff --git a/src/doveadm/dsync/dsync-ibc-pipe.c b/src/doveadm/dsync/dsync-ibc-pipe.c index 16b573c2a60..c8a1841d567 100644 --- a/src/doveadm/dsync/dsync-ibc-pipe.c +++ b/src/doveadm/dsync/dsync-ibc-pipe.c @@ -45,6 +45,7 @@ struct item { struct { const char *error; enum mail_error mail_error; + bool require_full_resync; } finish; } u; }; @@ -491,7 +492,8 @@ dsync_ibc_pipe_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r) static void dsync_ibc_pipe_send_finish(struct dsync_ibc *ibc, const char *error, - enum mail_error mail_error) + enum mail_error mail_error, + bool require_full_resync) { struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; struct item *item; @@ -499,11 +501,13 @@ dsync_ibc_pipe_send_finish(struct dsync_ibc *ibc, const char *error, item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_FINISH); item->u.finish.error = p_strdup(item->pool, error); item->u.finish.mail_error = mail_error; + item->u.finish.require_full_resync = require_full_resync; } static enum dsync_ibc_recv_ret dsync_ibc_pipe_recv_finish(struct dsync_ibc *ibc, const char **error_r, - enum mail_error *mail_error_r) + enum mail_error *mail_error_r, + bool *require_full_resync_r) { struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; struct item *item; @@ -514,6 +518,7 @@ dsync_ibc_pipe_recv_finish(struct dsync_ibc *ibc, const char **error_r, *error_r = item->u.finish.error; *mail_error_r = item->u.finish.mail_error; + *require_full_resync_r = item->u.finish.require_full_resync; return DSYNC_IBC_RECV_RET_OK; } diff --git a/src/doveadm/dsync/dsync-ibc-private.h b/src/doveadm/dsync/dsync-ibc-private.h index 62112c0e9ca..9ee1d384558 100644 --- a/src/doveadm/dsync/dsync-ibc-private.h +++ b/src/doveadm/dsync/dsync-ibc-private.h @@ -69,10 +69,12 @@ struct dsync_ibc_vfuncs { struct dsync_mail **mail_r); void (*send_finish)(struct dsync_ibc *ibc, const char *error, - enum mail_error mail_error); + enum mail_error mail_error, + bool require_full_resync); enum dsync_ibc_recv_ret (*recv_finish)(struct dsync_ibc *ibc, const char **error_r, - enum mail_error *mail_error_r); + enum mail_error *mail_error_r, + bool *require_full_resync_r); void (*close_mail_streams)(struct dsync_ibc *ibc); bool (*is_send_queue_full)(struct dsync_ibc *ibc); diff --git a/src/doveadm/dsync/dsync-ibc-stream.c b/src/doveadm/dsync/dsync-ibc-stream.c index 526f9d5bf50..a042fffaa0e 100644 --- a/src/doveadm/dsync/dsync-ibc-stream.c +++ b/src/doveadm/dsync/dsync-ibc-stream.c @@ -125,7 +125,7 @@ static const struct { }, { .name = "finish", .chr = 'F', - .optional_keys = "error mail_error", + .optional_keys = "error mail_error require_full_resync", .min_minor_version = DSYNC_PROTOCOL_MINOR_HAVE_FINISH }, { .name = "mailbox_cache_field", @@ -166,6 +166,8 @@ struct dsync_ibc_stream { unsigned int version_received:1; unsigned int handshake_received:1; unsigned int has_pending_data:1; + unsigned int finish_received:1; + unsigned int done_received:1; unsigned int stopped:1; }; @@ -366,10 +368,22 @@ static void dsync_ibc_stream_deinit(struct dsync_ibc *_ibc) if (ibc->value_output != NULL) i_stream_unref(&ibc->value_output); else { - /* notify remote that we're closing. this is mainly to avoid - "read() failed: EOF" errors on failing dsyncs */ - o_stream_nsend_str(ibc->output, - t_strdup_printf("%c\n", items[ITEM_DONE].chr)); + /* If the remote has not told us that they are closing we + notify remote that we're closing. this is mainly to avoid + "read() failed: EOF" errors on failing dsyncs. + + Avoid a race condition: + We do not tell the remote we are closing if they have + already told us because they close the + connection after sending ITEM_DONE and will + not be ever receive anything else from us unless + it just happens to get combined into the same packet + as a previous response and is already in the buffer. + */ + if (!ibc->done_received && !ibc->finish_received) { + o_stream_nsend_str(ibc->output, + t_strdup_printf("%c\n", items[ITEM_DONE].chr)); + } (void)o_stream_nfinish(ibc->output); } @@ -594,6 +608,7 @@ dsync_ibc_stream_input_next(struct dsync_ibc_stream *ibc, enum item_type item, /* remote cleanly closed the connection, possibly because of some failure (which it should have logged). we don't want to log any stream errors anyway after this. */ + ibc->done_received = TRUE; dsync_ibc_stream_stop(ibc); return DSYNC_IBC_RECV_RET_TRYAGAIN; @@ -1891,7 +1906,8 @@ dsync_ibc_stream_recv_mail(struct dsync_ibc *_ibc, struct dsync_mail **mail_r) static void dsync_ibc_stream_send_finish(struct dsync_ibc *_ibc, const char *error, - enum mail_error mail_error) + enum mail_error mail_error, + bool require_full_resync) { struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; struct dsync_serializer_encoder *encoder; @@ -1905,13 +1921,16 @@ dsync_ibc_stream_send_finish(struct dsync_ibc *_ibc, const char *error, dsync_serializer_encode_add(encoder, "mail_error", dec2str(mail_error)); } + if (require_full_resync) + dsync_serializer_encode_add(encoder, "require_full_resync", ""); dsync_serializer_encode_finish(&encoder, str); dsync_ibc_stream_send_string(ibc, str); } static enum dsync_ibc_recv_ret dsync_ibc_stream_recv_finish(struct dsync_ibc *_ibc, const char **error_r, - enum mail_error *mail_error_r) + enum mail_error *mail_error_r, + bool *require_full_resync_r) { struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; struct dsync_deserializer_decoder *decoder; @@ -1921,6 +1940,7 @@ dsync_ibc_stream_recv_finish(struct dsync_ibc *_ibc, const char **error_r, *error_r = NULL; *mail_error_r = 0; + *require_full_resync_r = FALSE; p_clear(ibc->ret_pool); @@ -1938,7 +1958,11 @@ dsync_ibc_stream_recv_finish(struct dsync_ibc *_ibc, const char **error_r, dsync_ibc_input_error(ibc, decoder, "Invalid mail_error"); return DSYNC_IBC_RECV_RET_TRYAGAIN; } + if (dsync_deserializer_decode_try(decoder, "require_full_resync", &value)) + *require_full_resync_r = TRUE; *mail_error_r = i; + + ibc->finish_received = TRUE; return DSYNC_IBC_RECV_RET_OK; } diff --git a/src/doveadm/dsync/dsync-ibc.c b/src/doveadm/dsync/dsync-ibc.c index e9eea93109d..f85bed30cb7 100644 --- a/src/doveadm/dsync/dsync-ibc.c +++ b/src/doveadm/dsync/dsync-ibc.c @@ -196,16 +196,19 @@ dsync_ibc_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r) } void dsync_ibc_send_finish(struct dsync_ibc *ibc, const char *error, - enum mail_error mail_error) + enum mail_error mail_error, + bool require_full_resync) { - ibc->v.send_finish(ibc, error, mail_error); + ibc->v.send_finish(ibc, error, mail_error, require_full_resync); } enum dsync_ibc_recv_ret dsync_ibc_recv_finish(struct dsync_ibc *ibc, const char **error_r, - enum mail_error *mail_error_r) + enum mail_error *mail_error_r, + bool *require_full_resync_r) { - return ibc->v.recv_finish(ibc, error_r, mail_error_r); + return ibc->v.recv_finish(ibc, error_r, mail_error_r, + require_full_resync_r); } void dsync_ibc_close_mail_streams(struct dsync_ibc *ibc) diff --git a/src/doveadm/dsync/dsync-ibc.h b/src/doveadm/dsync/dsync-ibc.h index d35d9211ecc..66b4026ceaf 100644 --- a/src/doveadm/dsync/dsync-ibc.h +++ b/src/doveadm/dsync/dsync-ibc.h @@ -148,10 +148,12 @@ enum dsync_ibc_recv_ret dsync_ibc_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r); void dsync_ibc_send_finish(struct dsync_ibc *ibc, const char *error, - enum mail_error mail_error); + enum mail_error mail_error, + bool require_full_resync); enum dsync_ibc_recv_ret dsync_ibc_recv_finish(struct dsync_ibc *ibc, const char **error_r, - enum mail_error *mail_error_r); + enum mail_error *mail_error_r, + bool *require_full_resync_r); /* Close any mail input streams that are kept open. This needs to be called before the mail is attempted to be freed (usually on error conditions). */ diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c index b67febc89b1..10dc3571f2b 100644 --- a/src/doveadm/dsync/dsync-mailbox-import.c +++ b/src/doveadm/dsync/dsync-mailbox-import.c @@ -108,6 +108,7 @@ struct dsync_mailbox_importer { enum mail_error mail_error; unsigned int failed:1; + unsigned int require_full_resync:1; unsigned int debug:1; unsigned int stateful_import:1; unsigned int last_common_uid_found:1; @@ -123,6 +124,10 @@ struct dsync_mailbox_importer { unsigned int delete_mailbox:1; }; +static const char *dsync_mail_change_type_names[] = { + "save", "expunge", "flag-change" +}; + static bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer, const struct dsync_mail *mail, struct importer_new_mail *all_newmails, @@ -145,6 +150,21 @@ imp_debug(struct dsync_mailbox_importer *importer, const char *fmt, ...) } T_END; } +static void +dsync_import_unexpected_state(struct dsync_mailbox_importer *importer, + const char *error) +{ + if (!importer->stateful_import) { + i_error("Mailbox %s: %s", mailbox_get_vname(importer->box), + error); + } else { + i_warning("Mailbox %s doesn't match previous state: %s " + "(dsync must be run again without the state)", + mailbox_get_vname(importer->box), error); + } + importer->require_full_resync = TRUE; +} + static void dsync_mailbox_import_search_init(struct dsync_mailbox_importer *importer) { @@ -267,6 +287,24 @@ dsync_mailbox_import_init(struct mailbox *box, importer->local_initial_highestpvtmodseq = status.highest_pvt_modseq; dsync_mailbox_import_search_init(importer); + if (!importer->stateful_import) + ; + else if (importer->local_uid_next <= last_common_uid) { + dsync_import_unexpected_state(importer, t_strdup_printf( + "local UIDNEXT %u <= last common UID %u", + importer->local_uid_next, last_common_uid)); + } else if (importer->local_initial_highestmodseq < last_common_modseq) { + dsync_import_unexpected_state(importer, t_strdup_printf( + "local HIGHESTMODSEQ %llu < last common HIGHESTMODSEQ %llu", + (unsigned long long)importer->local_initial_highestmodseq, + (unsigned long long)last_common_modseq)); + } else if (importer->local_initial_highestpvtmodseq < last_common_pvt_modseq) { + dsync_import_unexpected_state(importer, t_strdup_printf( + "local HIGHESTMODSEQ %llu < last common HIGHESTMODSEQ %llu", + (unsigned long long)importer->local_initial_highestpvtmodseq, + (unsigned long long)last_common_pvt_modseq)); + } + importer->local_changes = dsync_transaction_log_scan_get_hash(log_scan); importer->local_attr_changes = dsync_transaction_log_scan_get_attr_hash(log_scan); return importer; @@ -489,7 +527,7 @@ dsync_mailbox_import_attribute_real(struct dsync_mailbox_importer *importer, attr->key, &value) < 0) { i_error("Mailbox %s: Failed to set attribute %s: %s", mailbox_get_vname(importer->box), attr->key, - mailbox_get_last_error(importer->box, &importer->mail_error)); + mailbox_get_last_error(importer->box, NULL)); /* the attributes aren't vital, don't fail everything just because of them. */ } @@ -751,9 +789,10 @@ static bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer, /* add a record for local mail */ i_assert(importer->cur_mail != NULL); if (importer->revert_local_changes) { - if (save_change == NULL) { + if (save_change == NULL && + importer->cur_mail->uid >= importer->remote_uid_next) { dsync_mailbox_revert_existing_uid(importer, importer->cur_mail->uid, - t_strdup_printf("highest than remote's UIDs (remote UIDNEXT=%u)", importer->remote_uid_next)); + t_strdup_printf("higher than remote's UIDs (remote UIDNEXT=%u)", importer->remote_uid_next)); return TRUE; } mail_expunge(importer->cur_mail); @@ -829,20 +868,6 @@ static void dsync_mailbox_save(struct dsync_mailbox_importer *importer, while (!dsync_mailbox_try_save(importer, save_change)) ; } -static void -dsync_import_unexpected_state(struct dsync_mailbox_importer *importer, - const char *error) -{ - if (!importer->stateful_import) { - i_error("Mailbox %s: %s", mailbox_get_vname(importer->box), - error); - } else { - i_warning("Mailbox %s doesn't match previous state: %s " - "(dsync must be run again without the state)", - mailbox_get_vname(importer->box), error); - } -} - static bool dsync_import_set_mail(struct dsync_mailbox_importer *importer, const struct dsync_mail_change *change) @@ -871,8 +896,6 @@ dsync_import_set_mail(struct dsync_mailbox_importer *importer, dsync_import_unexpected_state(importer, t_strdup_printf( "Unexpected GUID mismatch for UID=%u: %s != %s", change->uid, guid, cmp_guid)); - importer->mail_error = MAIL_ERROR_TEMP; - importer->failed = TRUE; return FALSE; } return TRUE; @@ -891,8 +914,6 @@ static bool dsync_check_cur_guid(struct dsync_mailbox_importer *importer, dsync_import_unexpected_state(importer, t_strdup_printf( "Unexpected GUID mismatch (2) for UID=%u: %s != %s", change->uid, importer->cur_guid, cmp_guid)); - importer->mail_error = MAIL_ERROR_TEMP; - importer->failed = TRUE; return FALSE; } return TRUE; @@ -1667,7 +1688,7 @@ dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer, (void)dsync_mailbox_find_common_expunged_uid(importer, change, result_r); } *result_r = t_strdup_printf("%s (next local mail UID=%u)", - *result_r, importer->cur_mail->uid); + *result_r, importer->cur_mail == NULL ? 0 : importer->cur_mail->uid); } int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer, @@ -1682,6 +1703,8 @@ int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer, if (importer->failed) return -1; + if (importer->require_full_resync) + return 0; if (!importer->last_common_uid_found) { result = NULL; @@ -1691,12 +1714,15 @@ int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer, result = "New mail"; } - imp_debug(importer, "Import change GUID=%s UID=%u hdr_hash=%s result=%s", + imp_debug(importer, "Import change type=%s GUID=%s UID=%u hdr_hash=%s result=%s", + dsync_mail_change_type_names[change->type], change->guid != NULL ? change->guid : "", change->uid, change->hdr_hash != NULL ? change->hdr_hash : "", result); if (importer->failed) return -1; + if (importer->require_full_resync) + return 0; if (importer->last_common_uid_found) { /* a) uid <= last_common_uid for flag changes and expunges. @@ -2421,6 +2447,8 @@ int dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer, if (importer->failed) return -1; + if (importer->require_full_resync) + return 0; imp_debug(importer, "Import mail body for GUID=%s UID=%u", mail->guid, mail->uid); @@ -2734,6 +2762,7 @@ int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **_importer, uint64_t *last_common_pvt_modseq_r, uint32_t *last_messages_count_r, bool *changes_during_sync_r, + bool *require_full_resync_r, enum mail_error *error_r) { struct dsync_mailbox_importer *importer = *_importer; @@ -2742,8 +2771,9 @@ int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **_importer, *_importer = NULL; *changes_during_sync_r = FALSE; + *require_full_resync_r = importer->require_full_resync; - if (!success && !importer->failed) { + if ((!success || importer->require_full_resync) && !importer->failed) { importer->mail_error = MAIL_ERROR_TEMP; importer->failed = TRUE; } diff --git a/src/doveadm/dsync/dsync-mailbox-import.h b/src/doveadm/dsync/dsync-mailbox-import.h index d3603ea6215..8139d650ede 100644 --- a/src/doveadm/dsync/dsync-mailbox-import.h +++ b/src/doveadm/dsync/dsync-mailbox-import.h @@ -49,6 +49,7 @@ int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **importer, uint64_t *last_common_pvt_modseq_r, uint32_t *last_messages_count_r, bool *changes_during_sync_r, + bool *require_full_resync_r, enum mail_error *error_r); const char *dsync_mailbox_import_get_proctitle(struct dsync_mailbox_importer *importer); diff --git a/src/doveadm/dsync/dsync-mailbox-tree-sync.c b/src/doveadm/dsync/dsync-mailbox-tree-sync.c index 64fa3f7adb0..3b0937927a4 100644 --- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c @@ -15,8 +15,6 @@ #define TEMP_SUFFIX_MAX_LEN (sizeof("temp-")-1 + 8) #define TEMP_SUFFIX_FORMAT "temp-%x" -#define MAX_RENAMES 100 - struct dsync_mailbox_tree_bfs_iter { struct dsync_mailbox_tree *tree; @@ -27,13 +25,14 @@ struct dsync_mailbox_tree_bfs_iter { struct dsync_mailbox_tree_sync_ctx { pool_t pool; - struct dsync_brain *brain; struct dsync_mailbox_tree *local_tree, *remote_tree; enum dsync_mailbox_trees_sync_type sync_type; enum dsync_mailbox_trees_sync_flags sync_flags; + unsigned int combined_mailboxes_count; ARRAY(struct dsync_mailbox_tree_sync_change) changes; unsigned int change_idx; + bool failed; }; static struct dsync_mailbox_tree_bfs_iter * @@ -237,6 +236,7 @@ sync_tree_sort_and_delete_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, sync_set_node_deleted(tree, node); } } + ctx->combined_mailboxes_count++; array_append(&siblings, &node, 1); } sort_siblings(&siblings); @@ -1039,12 +1039,12 @@ sync_rename_delete_node_dirs(struct dsync_mailbox_tree_sync_ctx *ctx, static bool sync_rename_temp_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, struct dsync_mailbox_tree *tree, - struct dsync_mailbox_node *node) + struct dsync_mailbox_node *node, bool *renames_r) { const char *reason; for (; node != NULL; node = node->next) { - while (sync_rename_temp_mailboxes(ctx, tree, node->first_child)) ; + while (sync_rename_temp_mailboxes(ctx, tree, node->first_child, renames_r)) ; if (!node->sync_temporary_name) { } else if (dsync_mailbox_node_is_dir(node) && @@ -1061,6 +1061,7 @@ sync_rename_temp_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, sync_rename_delete_node_dirs(ctx, tree, node); } else { T_BEGIN { + *renames_r = TRUE; sync_rename_temp_mailbox_node(tree, node, &reason); if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0) { i_debug("brain %c: %s mailbox %s: %s", @@ -1075,12 +1076,16 @@ sync_rename_temp_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, return FALSE; } -static void -dsync_mailbox_tree_handle_renames(struct dsync_mailbox_tree_sync_ctx *ctx) +static int +dsync_mailbox_tree_handle_renames(struct dsync_mailbox_tree_sync_ctx *ctx, + bool *renames_r) { - unsigned int count = 0; + unsigned int max_renames, count = 0; bool changed; + *renames_r = FALSE; + + max_renames = ctx->combined_mailboxes_count * 3; do { T_BEGIN { changed = sync_rename_mailboxes(ctx, &ctx->local_tree->root, @@ -1091,15 +1096,16 @@ dsync_mailbox_tree_handle_renames(struct dsync_mailbox_tree_sync_ctx *ctx) i_debug("brain %c: -- Mailbox renamed, restart sync --", (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S'); } - } while (changed && ++count <= MAX_RENAMES); + } while (changed && ++count <= max_renames); if (changed) { i_error("BUG: Mailbox renaming algorithm got into a potentially infinite loop, aborting"); - ctx->brain->failed = TRUE; + return -1; } - while (sync_rename_temp_mailboxes(ctx, ctx->local_tree, &ctx->local_tree->root)) ; - while (sync_rename_temp_mailboxes(ctx, ctx->remote_tree, &ctx->remote_tree->root)) ; + while (sync_rename_temp_mailboxes(ctx, ctx->local_tree, &ctx->local_tree->root, renames_r)) ; + while (sync_rename_temp_mailboxes(ctx, ctx->remote_tree, &ctx->remote_tree->root, renames_r)) ; + return 0; } static bool sync_is_wrong_mailbox(struct dsync_mailbox_node *node, @@ -1389,6 +1395,8 @@ dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree, enum dsync_mailbox_trees_sync_flags sync_flags) { struct dsync_mailbox_tree_sync_ctx *ctx; + unsigned int rename_counter = 0; + bool renames; pool_t pool; i_assert(hash_table_is_created(local_tree->guid_hash)); @@ -1404,6 +1412,9 @@ dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree, ctx->sync_flags = sync_flags; i_array_init(&ctx->changes, 128); +again: + renames = FALSE; + ctx->combined_mailboxes_count = 0; sync_tree_sort_and_delete_mailboxes(ctx, remote_tree, sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY); sync_tree_sort_and_delete_mailboxes(ctx, local_tree, @@ -1411,8 +1422,12 @@ dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree, dsync_mailbox_tree_update_child_timestamps(&local_tree->root, 0); dsync_mailbox_tree_update_child_timestamps(&remote_tree->root, 0); - if ((sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES) == 0) - dsync_mailbox_tree_handle_renames(ctx); + if ((sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES) == 0) { + if (dsync_mailbox_tree_handle_renames(ctx, &renames) < 0) { + ctx->failed = TRUE; + return ctx; + } + } /* if we're not doing a two-way sync, delete now any mailboxes, which a) shouldn't exist, b) doesn't have a matching GUID/UIDVALIDITY, @@ -1426,6 +1441,14 @@ dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree, sync_create_mailboxes(ctx, remote_tree); if (sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) sync_create_mailboxes(ctx, local_tree); + if (renames && rename_counter++ <= ctx->combined_mailboxes_count*3) { + /* this rename algorithm is just horrible. we're retrying this + because the final sync_rename_temp_mailbox_node() calls + give different names to local & remote mailbox trees. + something's not right here, but this looping is better than + a crash in sync_mailbox_dirs() due to trees not matching. */ + goto again; + } sync_mailbox_dirs(ctx); return ctx; } @@ -1438,12 +1461,14 @@ dsync_mailbox_trees_sync_next(struct dsync_mailbox_tree_sync_ctx *ctx) return array_idx(&ctx->changes, ctx->change_idx++); } -void dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **_ctx) +int dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **_ctx) { struct dsync_mailbox_tree_sync_ctx *ctx = *_ctx; + int ret = ctx->failed ? -1 : 0; *_ctx = NULL; array_free(&ctx->changes); pool_unref(&ctx->pool); + return ret; } diff --git a/src/doveadm/dsync/dsync-mailbox-tree.h b/src/doveadm/dsync/dsync-mailbox-tree.h index af6ce369b99..d317aaa34f0 100644 --- a/src/doveadm/dsync/dsync-mailbox-tree.h +++ b/src/doveadm/dsync/dsync-mailbox-tree.h @@ -194,7 +194,7 @@ dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree, enum dsync_mailbox_trees_sync_flags sync_flags); const struct dsync_mailbox_tree_sync_change * dsync_mailbox_trees_sync_next(struct dsync_mailbox_tree_sync_ctx *ctx); -void dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **ctx); +int dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **ctx); const char *dsync_mailbox_node_to_string(const struct dsync_mailbox_node *node); const char * diff --git a/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c b/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c index af416db5546..1defcb2868a 100644 --- a/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c +++ b/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c @@ -703,6 +703,28 @@ static void test_dsync_mailbox_tree_sync_renames21(void) #endif } +static void test_dsync_mailbox_tree_sync_renames22(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 22"); + tree1 = dsync_mailbox_tree_init('/', '_'); + tree2 = dsync_mailbox_tree_init('/', '_'); + + node_create(tree1, 3, "p/a", 0); + node_create(tree1, 0, "p/2", 0); + node_create(tree1, 5, "p/2/h", 0); + + node_create(tree2, 4, "p/1/z", 0); + node_create(tree2, 1, "p/2", 0); + node_create(tree2, 2, "p/2/a", 0); + node_create(tree2, 5, "p/2/y", 0); + node_create(tree2, 3, "p/3", 0); + + test_trees(tree1, tree2); + test_end(); +} + static void test_dsync_mailbox_tree_sync_random(void) { struct dsync_mailbox_tree *tree1, *tree2; @@ -740,6 +762,7 @@ int main(void) test_dsync_mailbox_tree_sync_renames19, test_dsync_mailbox_tree_sync_renames20, test_dsync_mailbox_tree_sync_renames21, + test_dsync_mailbox_tree_sync_renames22, test_dsync_mailbox_tree_sync_random, NULL }; diff --git a/src/doveadm/main.c b/src/doveadm/main.c index dca58467425..aca450dba56 100644 --- a/src/doveadm/main.c +++ b/src/doveadm/main.c @@ -73,6 +73,7 @@ static void main_init(void) doveadm_http_server_init(); doveadm_cmds_init(); + doveadm_register_auth_server_commands(); doveadm_dump_init(); doveadm_mail_init(); dict_drivers_register_builtin(); @@ -100,6 +101,8 @@ int main(int argc, char *argv[]) }; enum master_service_flags service_flags = MASTER_SERVICE_FLAG_KEEP_CONFIG_OPEN; + struct master_service_settings_input input; + struct master_service_settings_output output; const char *error; int c; @@ -116,8 +119,13 @@ int main(int argc, char *argv[]) } } - if (master_service_settings_read_simple(master_service, set_roots, - &error) < 0) + memset(&input, 0, sizeof(input)); + input.roots = set_roots; + input.module = "doveadm"; + input.service = "doveadm"; + + if (master_service_settings_read(master_service, &input, &output, + &error) < 0) i_fatal("Error reading configuration: %s", error); master_service_init_log(master_service, "doveadm: "); diff --git a/src/imap/imap-client.c b/src/imap/imap-client.c index 4144f33bbd4..b5046015643 100644 --- a/src/imap/imap-client.c +++ b/src/imap/imap-client.c @@ -480,6 +480,7 @@ client_cmd_append_timing_stats(struct client_command_context *cmd, { unsigned int msecs_in_cmd, msecs_in_ioloop; uint64_t ioloop_wait_usecs; + unsigned int msecs_since_cmd; if (cmd->start_time.tv_sec == 0) return; @@ -488,12 +489,19 @@ client_cmd_append_timing_stats(struct client_command_context *cmd, msecs_in_cmd = (cmd->running_usecs + 999) / 1000; msecs_in_ioloop = (ioloop_wait_usecs - cmd->start_ioloop_wait_usecs + 999) / 1000; + msecs_since_cmd = timeval_diff_msecs(&ioloop_timeval, + &cmd->last_run_timeval); if (str_data(str)[str_len(str)-1] == '.') str_truncate(str, str_len(str)-1); - str_printfa(str, " (%d.%03d + %d.%03d secs).", + str_printfa(str, " (%d.%03d + %d.%03d ", msecs_in_cmd / 1000, msecs_in_cmd % 1000, msecs_in_ioloop / 1000, msecs_in_ioloop % 1000); + if (msecs_since_cmd > 0) { + str_printfa(str, "+ %d.%03d ", + msecs_since_cmd / 1000, msecs_since_cmd % 1000); + } + str_append(str, "secs)."); } void client_send_tagline(struct client_command_context *cmd, const char *data) @@ -506,6 +514,7 @@ void client_send_tagline(struct client_command_context *cmd, const char *data) i_assert(!cmd->tagline_sent); cmd->tagline_sent = TRUE; + cmd->tagline_reply = p_strdup(cmd->pool, data); if (tag == NULL || *tag == '\0') tag = "*"; @@ -583,9 +592,6 @@ bool client_read_args(struct client_command_context *cmd, unsigned int count, str = t_str_new(256); imap_write_args(str, *args_r); cmd->args = p_strdup(cmd->pool, str_c(str)); - cmd->start_time = ioloop_timeval; - cmd->start_ioloop_wait_usecs = - io_loop_get_wait_usecs(current_ioloop); cmd->client->input_lock = NULL; return TRUE; @@ -722,6 +728,9 @@ struct client_command_context *client_command_alloc(struct client *client) cmd = p_new(client->command_pool, struct client_command_context, 1); cmd->client = client; cmd->pool = client->command_pool; + cmd->start_time = ioloop_timeval; + cmd->last_run_timeval = ioloop_timeval; + cmd->start_ioloop_wait_usecs = io_loop_get_wait_usecs(current_ioloop); p_array_init(&cmd->module_contexts, cmd->pool, 5); DLLIST_PREPEND(&client->command_queue, cmd); diff --git a/src/imap/imap-client.h b/src/imap/imap-client.h index 82f2a3987b3..9b5415e26f2 100644 --- a/src/imap/imap-client.h +++ b/src/imap/imap-client.h @@ -66,6 +66,7 @@ struct client_command_context { them. */ const char *args; enum command_flags cmd_flags; + const char *tagline_reply; command_func_t *func; void *context; @@ -78,6 +79,9 @@ struct client_command_context { /* time when command handling was started - typically this is after reading all the parameters. */ struct timeval start_time; + /* time when command handling was last finished. this is before + mailbox syncing is done. */ + struct timeval last_run_timeval; /* io_loop_get_wait_usecs()'s value when the command was started */ uint64_t start_ioloop_wait_usecs; /* how many usecs this command itself has spent running */ diff --git a/src/imap/imap-settings.c b/src/imap/imap-settings.c index d1668e6910a..d3e869cf806 100644 --- a/src/imap/imap-settings.c +++ b/src/imap/imap-settings.c @@ -5,6 +5,7 @@ #include "settings-parser.h" #include "service-settings.h" #include "mail-storage-settings.h" +#include "lda-settings.h" #include "imap-settings.h" #include @@ -101,6 +102,7 @@ static const struct imap_settings imap_default_settings = { static const struct setting_parser_info *imap_setting_dependencies[] = { &mail_user_setting_parser_info, + &lda_setting_parser_info, NULL }; diff --git a/src/imap/imap-sync.c b/src/imap/imap-sync.c index df70625380c..27f31d05430 100644 --- a/src/imap/imap-sync.c +++ b/src/imap/imap-sync.c @@ -315,7 +315,7 @@ static int imap_sync_finish(struct imap_sync_context *ctx, bool aborting) if (mailbox_sync_deinit(&ctx->sync_ctx, &ctx->sync_status) < 0 || ctx->failed) { ctx->failed = TRUE; - return -1; + ret = -1; } mailbox_get_open_status(ctx->box, STATUS_UIDVALIDITY | STATUS_MESSAGES | STATUS_RECENT | @@ -762,18 +762,20 @@ bool cmd_sync(struct client_command_context *cmd, enum mailbox_sync_flags flags, if (cmd->cancel) return TRUE; + cmd->last_run_timeval = ioloop_timeval; if (client->mailbox == NULL) { /* no mailbox selected, no point in delaying the sync */ if (tagline != NULL) client_send_tagline(cmd, tagline); return TRUE; } + cmd->tagline_reply = p_strdup(cmd->pool, tagline); cmd->sync = p_new(cmd->pool, struct client_sync_context, 1); cmd->sync->counter = client->sync_counter; cmd->sync->flags = flags; cmd->sync->imap_flags = imap_flags; - cmd->sync->tagline = p_strdup(cmd->pool, tagline); + cmd->sync->tagline = cmd->tagline_reply; cmd->state = CLIENT_COMMAND_STATE_WAIT_SYNC; cmd->func = NULL; diff --git a/src/lib-auth/auth-master.c b/src/lib-auth/auth-master.c index a138417e206..b3ceab510f8 100644 --- a/src/lib-auth/auth-master.c +++ b/src/lib-auth/auth-master.c @@ -413,7 +413,8 @@ static int auth_master_run_cmd_pre(struct auth_master_connection *conn, o_stream_uncork(conn->output); if (o_stream_nfinish(conn->output) < 0) { - i_error("write(auth socket) failed: %m"); + i_error("write(auth socket) failed: %s", + o_stream_get_error(conn->output)); auth_master_unset_io(conn); auth_connection_close(conn); return -1; diff --git a/src/lib-auth/auth-server-connection.c b/src/lib-auth/auth-server-connection.c index 528d49a6714..8453a3a61f4 100644 --- a/src/lib-auth/auth-server-connection.c +++ b/src/lib-auth/auth-server-connection.c @@ -453,9 +453,10 @@ int auth_server_connection_connect(struct auth_server_connection *conn) AUTH_CLIENT_PROTOCOL_MINOR_VERSION, conn->client->client_pid); if (o_stream_send_str(conn->output, handshake) < 0) { - i_warning("Error sending handshake to auth server: %m"); + i_warning("Error sending handshake to auth server: %s", + o_stream_get_error(conn->output)); auth_server_connection_disconnect(conn, - strerror(conn->output->last_failed_errno)); + o_stream_get_error(conn->output)); return -1; } diff --git a/src/lib-charset/charset-iconv.c b/src/lib-charset/charset-iconv.c index 964ddec9388..85400a1836c 100644 --- a/src/lib-charset/charset-iconv.c +++ b/src/lib-charset/charset-iconv.c @@ -24,6 +24,8 @@ int charset_to_utf8_begin(const char *charset, normalizer_func_t *normalizer, if (charset_is_utf8(charset)) cd = (iconv_t)-1; else { + if (strcmp(charset, "UTF-8//TEST") == 0) + charset = "UTF-8"; cd = iconv_open("UTF-8", charset); if (cd == (iconv_t)-1) return -1; diff --git a/src/lib-charset/test-charset.c b/src/lib-charset/test-charset.c index 18da108464e..1f763737685 100644 --- a/src/lib-charset/test-charset.c +++ b/src/lib-charset/test-charset.c @@ -105,7 +105,7 @@ static void test_charset_iconv(void) } /* Use //IGNORE just to force handling to be done by iconv instead of our own UTF-8 routines. */ - test_charset_utf8_common("UTF-8//IGNORE"); + test_charset_utf8_common("UTF-8//TEST"); test_end(); } static void test_charset_iconv_crashes(void) diff --git a/src/lib-compression/istream-bzlib.c b/src/lib-compression/istream-bzlib.c index b8d0dff86cc..cb3db16ac8b 100644 --- a/src/lib-compression/istream-bzlib.c +++ b/src/lib-compression/istream-bzlib.c @@ -86,8 +86,7 @@ static ssize_t i_stream_bzlib_read(struct istream_private *stream) have a seek mark. */ i_stream_compress(stream); } - if (stream->max_buffer_size == 0 || - stream->buffer_size < stream->max_buffer_size) + if (stream->buffer_size < i_stream_get_max_buffer_size(&stream->istream)) i_stream_grow_buffer(stream, CHUNK_SIZE); if (stream->pos == stream->buffer_size) { diff --git a/src/lib-compression/istream-lz4.c b/src/lib-compression/istream-lz4.c index d53c970caa6..ded44dc8ce7 100644 --- a/src/lib-compression/istream-lz4.c +++ b/src/lib-compression/istream-lz4.c @@ -150,7 +150,7 @@ static ssize_t i_stream_lz4_read(struct istream_private *stream) } /* if we already have max_buffer_size amount of data, fail here */ i_stream_compress(stream); - if (stream->pos >= stream->max_buffer_size) + if (stream->pos >= i_stream_get_max_buffer_size(&stream->istream)) return -2; /* allocate enough space for the old data and the new decompressed chunk. we don't know the original compressed size, diff --git a/src/lib-compression/istream-lzma.c b/src/lib-compression/istream-lzma.c index 865a7819d55..20bb93b17ea 100644 --- a/src/lib-compression/istream-lzma.c +++ b/src/lib-compression/istream-lzma.c @@ -95,8 +95,7 @@ static ssize_t i_stream_lzma_read(struct istream_private *stream) have a seek mark. */ i_stream_compress(stream); } - if (stream->max_buffer_size == 0 || - stream->buffer_size < stream->max_buffer_size) + if (stream->buffer_size < i_stream_get_max_buffer_size(&stream->istream)) i_stream_grow_buffer(stream, CHUNK_SIZE); if (stream->pos == stream->buffer_size) { diff --git a/src/lib-compression/istream-zlib.c b/src/lib-compression/istream-zlib.c index d212a20ce2c..b32a1d73e9b 100644 --- a/src/lib-compression/istream-zlib.c +++ b/src/lib-compression/istream-zlib.c @@ -230,8 +230,7 @@ static ssize_t i_stream_zlib_read(struct istream_private *stream) have a seek mark. */ i_stream_compress(stream); } - if (stream->max_buffer_size == 0 || - stream->buffer_size < stream->max_buffer_size) + if (stream->buffer_size < i_stream_get_max_buffer_size(&stream->istream)) i_stream_grow_buffer(stream, CHUNK_SIZE); if (stream->pos == stream->buffer_size) { diff --git a/src/lib-dcrypt/Makefile.am b/src/lib-dcrypt/Makefile.am new file mode 100644 index 00000000000..946590e7b74 --- /dev/null +++ b/src/lib-dcrypt/Makefile.am @@ -0,0 +1,67 @@ +noinst_LTLIBRARIES = libdcrypt.la +pkglib_LTLIBRARIES = + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-ssl-iostream + +libdcrypt_la_SOURCES = \ + dcrypt.c \ + istream-decrypt.c \ + ostream-encrypt.c + +libdcrypt_la_CFLAGS = $(AM_CPPFLAGS) \ + -DDCRYPT_MODULE_DIR=\"$(pkglibdir)\" + +if BUILD_OPENSSL +pkglib_LTLIBRARIES += libdcrypt_openssl.la +libdcrypt_openssl_la_SOURCES = dcrypt-openssl.c dcrypt.c +libdcrypt_openssl_la_LDFLAGS = -module -avoid-version -shared ../lib-ssl-iostream/libdovecot_openssl_common.la ../lib/liblib.la +libdcrypt_openssl_la_DEPENDENCIES = ../lib-ssl-iostream/libdovecot_openssl_common.la ../lib/liblib.la +libdcrypt_openssl_la_CFLAGS = $(AM_CPPFLAGS) \ + -DDCRYPT_MODULE_DIR=\"$(pkglibdir)\" + $(SSL_CFLAGS) +endif + +headers = \ + dcrypt.h \ + dcrypt-iostream-private.h \ + dcrypt-private.h \ + ostream-encrypt.h \ + istream-decrypt.h + +pkginc_libdir=$(pkgincludedir) +pkginc_lib_HEADERS = $(headers) + +EXTRA_DIST = \ + sample-v1.asc \ + sample-v1_short.asc \ + sample-v2.asc + +test_programs = test-crypto test-stream +noinst_PROGRAMS = $(test_programs) + +check: check-am check-test + +check-test: all-am + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done + +LIBDOVECOT_TEST_DEPS = \ + ../lib-test/libtest.la \ + ../lib/liblib.la +LIBDOVECOT_TEST = \ + $(LIBDOVECOT_TEST_DEPS) \ + $(MODULE_LIBS) + +test_crypto_LDADD = $(LIBDOVECOT_TEST) +test_crypto_DEPENDENCIES = $(LIBDOVECOT_TEST_DEPS) +test_crypto_CFLAGS = $(AM_CPPFLAGS) -DDCRYPT_MODULE_DIR=\".libs\" -DDCRYPT_SRC_DIR=\"$(top_srcdir)/src/lib-dcrypt\" +test_crypto_SOURCES = $(libdcrypt_la_SOURCES) test-crypto.c + +test_stream_LDADD = $(LIBDOVECOT_TEST) +test_stream_DEPENDENCIES = $(LIBDOVECOT_TEST_DEPS) +test_stream_CFLAGS = $(AM_CPPFLAGS) -DDCRYPT_MODULE_DIR=\".libs\" -DDCRYPT_SRC_DIR=\"$(top_srcdir)/src/lib-dcrypt\" +test_stream_SOURCES = $(libdcrypt_la_SOURCES) test-stream.c diff --git a/src/lib-dcrypt/dcrypt-gnutls.c b/src/lib-dcrypt/dcrypt-gnutls.c new file mode 100644 index 00000000000..7185435aa45 --- /dev/null +++ b/src/lib-dcrypt/dcrypt-gnutls.c @@ -0,0 +1,493 @@ +#include "lib.h" +#include "buffer.h" +#include "randgen.h" +#include "array.h" +#include "hash-method.h" +#include "pkcs5.h" +#include "module-dir.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#include "dcrypt.h" +#include "dcrypt-private.h" + +struct dcrypt_context_symmetric { + pool_t pool; + gnutls_cipher_hd_t ctx; + gnutls_cipher_algorithm_t cipher; + gnutls_datum_t key; + gnutls_datum_t iv; + enum dcrypt_sym_mode mode; +}; + +struct dcrypt_context_hmac { + pool_t pool; + gnutls_hmac_hd_t ctx; + gnutls_mac_algorithm_t md; + gnutls_datum_t key; + size_t klen; +}; + +struct dcrypt_public_key { + void *ctx; +}; + +struct dcrypt_private_key { + void *ctx; +}; + +static +int dcrypt_gnutls_private_to_public_key(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r, const char **error_r); + +static +int dcrypt_gnutls_error(int ec, const char **error_r) +{ + i_assert(ec < 0); + if(error_r != NULL) { + *error_r = gnutls_strerror(ec); + } + return -1; +} + +static +int dcrypt_gnutls_ctx_sym_create(const char *algorithm, enum dcrypt_sym_mode mode, struct dcrypt_context_symmetric **ctx_r, const char **error_r) +{ + gnutls_cipher_algorithm_t cipher = gnutls_cipher_get_id(algorithm); + if(cipher == GNUTLS_CIPHER_UNKNOWN) return dcrypt_gnutls_error(cipher, error_r); + pool_t pool = pool_alloconly_create("dcrypt gnutls", 128); + struct dcrypt_context_symmetric *ctx = p_new(pool, struct dcrypt_context_symmetric, 1); + ctx->pool = pool; + ctx->cipher = cipher; + ctx->mode = mode; + *ctx_r = ctx; + return 0; +} + +static +int dcrypt_gnutls_ctx_sym_destroy(struct dcrypt_context_symmetric **ctx) +{ + pool_t pool =(*ctx)->pool; + gnutls_cipher_deinit((*ctx)->ctx); + pool_unref(&pool); + *ctx = NULL; + return 0; +} + +static +void dcrypt_gnutls_ctx_sym_set_key(struct dcrypt_context_symmetric *ctx, const unsigned char *key, size_t key_len) +{ + if(ctx->key.data != NULL) p_free(ctx->pool, ctx->key.data); + ctx->key.size = I_MIN(key_len,(size_t)gnutls_cipher_get_key_size(ctx->cipher)); + ctx->key.data = p_malloc(ctx->pool, ctx->key.size); + memcpy(ctx->key.data, key, ctx->key.size); +} + +static +void dcrypt_gnutls_ctx_sym_set_iv(struct dcrypt_context_symmetric *ctx, const unsigned char *iv, size_t iv_len) +{ + if(ctx->iv.data != NULL) p_free(ctx->pool, ctx->iv.data); + ctx->iv.size = I_MIN(iv_len,(size_t)gnutls_cipher_get_iv_size(ctx->cipher)); + ctx->iv.data = p_malloc(ctx->pool, ctx->iv.size); + memcpy(ctx->iv.data, iv, ctx->iv.size); +} + +static +void dcrypt_gnutls_ctx_sym_set_key_iv_random(struct dcrypt_context_symmetric *ctx) +{ + if(ctx->key.data != NULL) p_free(ctx->pool, ctx->key.data); + if(ctx->iv.data != NULL) p_free(ctx->pool, ctx->iv.data); + ctx->key.data = p_malloc(ctx->pool, gnutls_cipher_get_key_size(ctx->cipher)); + random_fill(ctx->key.data, gnutls_cipher_get_key_size(ctx->cipher)); + ctx->key.size = gnutls_cipher_get_key_size(ctx->cipher); + ctx->iv.data = p_malloc(ctx->pool, gnutls_cipher_get_iv_size(ctx->cipher)); + random_fill(ctx->iv.data, gnutls_cipher_get_iv_size(ctx->cipher)); + ctx->iv.size = gnutls_cipher_get_iv_size(ctx->cipher); +} + +static +int dcrypt_gnutls_ctx_sym_get_key(struct dcrypt_context_symmetric *ctx, buffer_t *key) +{ + if(ctx->key.data == NULL) return -1; + buffer_append(key, ctx->key.data, ctx->key.size); + return 0; +} +static +int dcrypt_gnutls_ctx_sym_get_iv(struct dcrypt_context_symmetric *ctx, buffer_t *iv) +{ + if(ctx->iv.data == NULL) return -1; + buffer_append(iv, ctx->iv.data, ctx->iv.size); + return 0; +} + +static +int dcrypt_gnutls_ctx_sym_get_key_length(struct dcrypt_context_symmetric *ctx) +{ + return gnutls_cipher_get_iv_size(ctx->cipher); +} +static +int dcrypt_gnutls_ctx_sym_get_iv_length(struct dcrypt_context_symmetric *ctx) +{ + return gnutls_cipher_get_iv_size(ctx->cipher); +} +static +int dcrypt_gnutls_ctx_sym_get_block_size(struct dcrypt_context_symmetric *ctx) +{ + return gnutls_cipher_get_block_size(ctx->cipher); +} + +static +int dcrypt_gnutls_ctx_sym_init(struct dcrypt_context_symmetric *ctx, const char **error_r) +{ + int ec; + ec = gnutls_cipher_init(&(ctx->ctx), ctx->cipher, &ctx->key, &ctx->iv); + if(ec < 0) return dcrypt_gnutls_error(ec, error_r); + return 0; +} + +static +int dcrypt_gnutls_ctx_sym_update(struct dcrypt_context_symmetric *ctx, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r) +{ + int ec; + size_t outl = gnutls_cipher_get_block_size(ctx->cipher); + unsigned char buf[outl]; + ec = gnutls_cipher_encrypt2(ctx->ctx, data, data_len, buf, outl); + if(ec < 0) return dcrypt_gnutls_error(ec, error_r); + buffer_append(result, buf, outl); + return ec; +} + +static +int dcrypt_gnutls_ctx_sym_final(struct dcrypt_context_symmetric *ctx, buffer_t *result, const char **error_r) +{ + return dcrypt_gnutls_ctx_sym_update(ctx, (const unsigned char*)"", 0, result, error_r); +} + + +static +int dcrypt_gnutls_ctx_hmac_create(const char *algorithm, struct dcrypt_context_hmac **ctx_r, const char **error_r) +{ + gnutls_mac_algorithm_t md = gnutls_mac_get_id(algorithm); + if (md == GNUTLS_MAC_UNKNOWN) return dcrypt_gnutls_error(md, error_r); + pool_t pool = pool_alloconly_create("dcrypt gnutls", 128); + struct dcrypt_context_hmac *ctx = p_new(pool, struct dcrypt_context_hmac, 1); + ctx->pool = pool; + ctx->md = md; + *ctx_r = ctx; + return 0; +} + +static +int dcrypt_gnutls_ctx_hmac_destroy(struct dcrypt_context_hmac **ctx) +{ + pool_t pool = (*ctx)->pool; + gnutls_hmac_deinit((*ctx)->ctx, NULL); + pool_unref(&pool); + *ctx = NULL; + return 0; +} + + +static +void dcrypt_gnutls_ctx_hmac_set_key(struct dcrypt_context_hmac *ctx, const unsigned char *key, size_t key_len) +{ + if(ctx->key.data != NULL) p_free(ctx->pool, ctx->key.data); + ctx->key.size = I_MIN(key_len,(size_t)gnutls_hmac_get_len(ctx->md)); + ctx->key.data = p_malloc(ctx->pool, ctx->key.size); + memcpy(ctx->key.data, key, ctx->key.size); +} + +static +int dcrypt_gnutls_ctx_hmac_get_key(struct dcrypt_context_hmac *ctx, buffer_t *key) +{ + if (ctx->key.data == NULL) return -1; + buffer_append(key, ctx->key.data, ctx->key.size); + return 0; +} + +static +int dcrypt_gnutls_ctx_hmac_init(struct dcrypt_context_hmac *ctx, const char **error_r) +{ + int ec; + ec = gnutls_hmac_init(&(ctx->ctx), ctx->md, ctx->key.data, ctx->key.size); + if (ec < 0) return dcrypt_gnutls_error(ec, error_r); + return 0; +} +static +int dcrypt_gnutls_ctx_hmac_update(struct dcrypt_context_hmac *ctx, const unsigned char *data, size_t data_len, const char **error_r) +{ + int ec; + if ((ec = gnutls_hmac(ctx->ctx, data, data_len)) != 0) + return dcrypt_gnutls_error(ec, error_r); + return 0; +} +static +int dcrypt_gnutls_ctx_hmac_final(struct dcrypt_context_hmac *ctx, buffer_t *result, const char **error_r) +{ + size_t hlen = gnutls_hmac_get_len(ctx->md); + unsigned char buf[hlen]; + gnutls_hmac_output(ctx->ctx, buf); + buffer_append(result, buf, hlen); + return 0; +} + +static +int dcrypt_gnutls_ecdh_derive_secret(struct dcrypt_public_key *peer_key, buffer_t *R, buffer_t *S, const char **error_r) +{ + +} + +static +int dcrypt_gnutls_pbkdf2(const unsigned char *password, size_t password_len, const unsigned char *salt, size_t salt_len, const char *algorithm, + unsigned int rounds, buffer_t *result, unsigned int result_len, const char **error_r) +{ + unsigned char buf[result_len]; + /* only sha1 or sha256 is supported */ + if (strncasecmp(algorithm, "sha1", 4) == 0) { + pbkdf2_hmac_sha1(password_len, password, rounds, salt_len, salt, result_len, buf); + } else if (strncasecmp(algorithm, "sha256", 6) == 0) { + pbkdf2_hmac_sha256(password_len, password, rounds, salt_len, salt, result_len, buf); + } else if (strncasecmp(algorithm, "sha512", 6) == 0) { + struct hmac_sha512_ctx ctx; + hmac_sha512_set_key(&ctx, password_len, password); + PBKDF2(&ctx, hmac_sha512_update, hmac_sha512_digest, 64, rounds, salt_len, salt, result_len, buf); + memset(&ctx, 0, sizeof(ctx)); + } else { + *error_r = "Unsupported algorithm"; + return -1; + } + buffer_append(result, buf, result_len); + memset(buf, 0, sizeof(buf)); + return 0; +} + +static +int dcrypt_gnutls_generate_keypair(struct dcrypt_keypair *pair_r, enum dcrypt_key_type kind, unsigned int bits, const char *curve, const char **error_r) +{ + gnutls_pk_algorithm_t pk_algo; + gnutls_ecc_curve_t pk_curve; + + if (kind == DCRYPT_KEY_EC) { + pk_curve = gnutls_ecc_curve_get_id(curve); + if (pk_curve == GNUTLS_ECC_CURVE_INVALID) { + *error_r = "Invalid curve"; + return -1; + } + bits = GNUTLS_CURVE_TO_BITS(pk_curve); +#if GNUTLS_VERSION_NUMBER >= 0x030500 + pk_algo = gnutls_curve_get_pk(pk_curve); +#else + pk_algo = GNUTLS_PK_EC; +#endif + } else if (kind == DCRYPT_KEY_RSA) { + pk_algo = gnutls_pk_get_id("RSA"); + } else { + *error_r = "Unsupported key type"; + return -1; + } + + int ec; + gnutls_privkey_t priv; + if ((ec = gnutls_privkey_init(&priv)) != GNUTLS_E_SUCCESS) return dcrypt_gnutls_error(ec, error_r); +#if GNUTLS_VERSION_NUMBER >= 0x030500 + gnutls_privkey_set_flags(priv, GNUTLS_PRIVKEY_FLAG_EXPORT_COMPAT); +#endif + ec = gnutls_privkey_generate(priv, pk_algo, bits, 0); + if (ec != GNUTLS_E_SUCCESS) { + gnutls_privkey_deinit(priv); + return dcrypt_gnutls_error(ec, error_r); + } + + pair_r->priv = (struct dcrypt_private_key*)priv; + + return dcrypt_gnutls_private_to_public_key(pair_r->priv, &(pair_r->pub), error_r); +} + +static +int dcrypt_gnutls_load_private_key(struct dcrypt_private_key **key_r, const unsigned char *data, size_t data_len, dcrypt_password_cb *cb, void *ctx, const char **error_r) +{ + +} +static +int dcrypt_gnutls_load_public_key(struct dcrypt_public_key **key_r, const unsigned char *data, size_t data_len, const char **error_r) +{ + +} + +static +int dcrypt_gnutls_store_private_key(struct dcrypt_private_key *key, const char *cipher, buffer_t *destination, dcrypt_password_cb *cb, void *ctx, const char **error_r) +{ + gnutls_privkey_t priv = (gnutls_privkey_t)key; + gnutls_x509_privkey_t xkey; + gnutls_privkey_export_x509(priv, &xkey); + /* then export PEM */ + size_t outl = 0; + gnutls_x509_privkey_export_pkcs8(xkey, GNUTLS_X509_FMT_PEM, NULL, 0, NULL, &outl); + char buffer[outl]; + gnutls_x509_privkey_export_pkcs8(xkey, GNUTLS_X509_FMT_PEM, NULL, 0, buffer, &outl); + buffer_append(destination, buffer, outl); + memset(buffer, 0, sizeof(buffer)); + return 0; +} + +static +int dcrypt_gnutls_store_public_key(struct dcrypt_public_key *key, buffer_t *destination, const char **error_r) +{ + gnutls_pubkey_t pub = (gnutls_pubkey_t)key; + size_t outl = 0; + gnutls_pubkey_export(pub, GNUTLS_X509_FMT_PEM, NULL, &outl); + char buffer[outl]; + gnutls_pubkey_export(pub, GNUTLS_X509_FMT_PEM, buffer, &outl); + buffer_append(destination, buffer, outl); + return 0; +} + +static +int dcrypt_gnutls_private_to_public_key(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r, const char **error_r) +{ + int ec; + + gnutls_privkey_t priv = (gnutls_privkey_t)priv_key; + if (gnutls_privkey_get_pk_algorithm(priv, NULL) == GNUTLS_PK_RSA) { + gnutls_datum_t m,e; + /* do not extract anything we don't need */ + ec = gnutls_privkey_export_rsa_raw(priv, &m, &e, NULL, NULL, NULL, NULL, NULL, NULL); + if (ec != GNUTLS_E_SUCCESS) return dcrypt_gnutls_error(ec, error_r); + gnutls_pubkey_t pub; + gnutls_pubkey_init(&pub); + ec = gnutls_pubkey_import_rsa_raw(pub, &m, &e); + gnutls_free(m.data); + gnutls_free(e.data); + if (ec < 0) { + gnutls_pubkey_deinit(pub); + return dcrypt_gnutls_error(ec, error_r); + } + *pub_key_r = (struct dcrypt_public_key*)pub; + return 0; + } else if (gnutls_privkey_get_pk_algorithm(priv, NULL) == GNUTLS_PK_EC) { + gnutls_ecc_curve_t curve; + gnutls_datum_t x,y,k; + ec = gnutls_privkey_export_ecc_raw(priv, &curve, &x, &y, NULL); + if (ec != GNUTLS_E_SUCCESS) return dcrypt_gnutls_error(ec, error_r); + gnutls_pubkey_t pub; + gnutls_pubkey_init(&pub); + ec = gnutls_pubkey_import_ecc_raw(pub, curve, &x, &y); + gnutls_free(x.data); + gnutls_free(y.data); + if (ec < 0) { + gnutls_pubkey_deinit(pub); + return dcrypt_gnutls_error(ec, error_r); + } + *pub_key_r = (struct dcrypt_public_key*)pub; + return 0; + } + + return -1; +} + +static +void dcrypt_gnutls_free_public_key(struct dcrypt_public_key **key) +{ + gnutls_pubkey_deinit((gnutls_pubkey_t)*key); + *key = NULL; +} +static +void dcrypt_gnutls_free_private_key(struct dcrypt_private_key **key) +{ + gnutls_privkey_deinit((gnutls_privkey_t)*key); + *key = NULL; +} +static +void dcrypt_gnutls_free_keypair(struct dcrypt_keypair *keypair) +{ + dcrypt_gnutls_free_public_key(&(keypair->pub)); + dcrypt_gnutls_free_private_key(&(keypair->priv)); +} + +static +int dcrypt_gnutls_rsa_encrypt(struct dcrypt_public_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r) +{ + +} +static +int dcrypt_gnutls_rsa_decrypt(struct dcrypt_private_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r) +{ + +} + +static +int dcrypt_gnutls_oid_keytype(const unsigned char *oid, size_t oid_len, enum dcrypt_key_type *key_type, const char **error_r) +{ + +} +static +int dcrypt_gnutls_keytype_oid(enum dcrypt_key_type key_type, buffer_t *oid, const char **error_r) +{ + +} + +static +const char *dcrypt_gnutls_oid2name(const unsigned char *oid, size_t oid_len, const char **error_r) +{ +} + +static +int dcrypt_gnutls_name2oid(const char *name, buffer_t *oid, const char **error_r) +{ + +} + +static struct dcrypt_vfs dcrypt_gnutls_vfs = { + .ctx_sym_create = dcrypt_gnutls_ctx_sym_create, + .ctx_sym_destroy = dcrypt_gnutls_ctx_sym_destroy, + .ctx_sym_set_key = dcrypt_gnutls_ctx_sym_set_key, + .ctx_sym_set_iv = dcrypt_gnutls_ctx_sym_set_iv, + .ctx_sym_set_key_iv_random = dcrypt_gnutls_ctx_sym_set_key_iv_random, + .ctx_sym_get_key = dcrypt_gnutls_ctx_sym_get_key, + .ctx_sym_get_iv = dcrypt_gnutls_ctx_sym_get_iv, + .ctx_sym_get_key_length = dcrypt_gnutls_ctx_sym_get_key_length, + .ctx_sym_get_iv_length = dcrypt_gnutls_ctx_sym_get_iv_length, + .ctx_sym_init = dcrypt_gnutls_ctx_sym_init, + .ctx_sym_update = dcrypt_gnutls_ctx_sym_update, + .ctx_sym_final = dcrypt_gnutls_ctx_sym_final, + .ctx_hmac_create = dcrypt_gnutls_ctx_hmac_create, + .ctx_hmac_destroy = dcrypt_gnutls_ctx_hmac_destroy, + .ctx_hmac_set_key = dcrypt_gnutls_ctx_hmac_set_key, + .ctx_hmac_get_key = dcrypt_gnutls_ctx_hmac_get_key, + .ctx_hmac_init = dcrypt_gnutls_ctx_hmac_init, + .ctx_hmac_update = dcrypt_gnutls_ctx_hmac_update, + .ctx_hmac_final = dcrypt_gnutls_ctx_hmac_final, +// .ecdh_derive_secret = dcrypt_gnutls_ecdh_derive_secret, + .pbkdf2 = dcrypt_gnutls_pbkdf2, + .generate_keypair = dcrypt_gnutls_generate_keypair, + .load_private_key = dcrypt_gnutls_load_private_key, + .load_public_key = dcrypt_gnutls_load_public_key, + .store_private_key = dcrypt_gnutls_store_private_key, + .store_public_key = dcrypt_gnutls_store_public_key, + .private_to_public_key = dcrypt_gnutls_private_to_public_key, + .free_keypair = dcrypt_gnutls_free_keypair, + .free_public_key = dcrypt_gnutls_free_public_key, + .free_private_key = dcrypt_gnutls_free_private_key, + .rsa_encrypt = dcrypt_gnutls_rsa_encrypt, + .rsa_decrypt = dcrypt_gnutls_rsa_decrypt, + .oid_keytype = dcrypt_gnutls_oid_keytype, + .keytype_oid = dcrypt_gnutls_keytype_oid, + .oid2name = dcrypt_gnutls_oid2name, + .name2oid = dcrypt_gnutls_name2oid +}; + +void dcrypt_gnutls_init(struct module *module ATTR_UNUSED) +{ + gnutls_global_init(); + dcrypt_set_vfs(&dcrypt_gnutls_vfs); +} + +void dcrypt_gnutls_deinit(void) +{ + gnutls_global_deinit(); +} diff --git a/src/lib-dcrypt/dcrypt-iostream-private.h b/src/lib-dcrypt/dcrypt-iostream-private.h new file mode 100644 index 00000000000..d5cb7f17bca --- /dev/null +++ b/src/lib-dcrypt/dcrypt-iostream-private.h @@ -0,0 +1,8 @@ +#ifndef DCRYPT_IOSTREAM_PRIVATE_H +#define DCRYPT_IOSTREAM_PRIVATE 1 + +static const unsigned char IOSTREAM_CRYPT_MAGIC[] = {'C','R','Y','P','T','E','D','\x03','\x07'}; +#define IOSTREAM_CRYPT_VERSION 2 +#define IOSTREAM_TAG_SIZE 16 + +#endif diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c new file mode 100644 index 00000000000..f967695e643 --- /dev/null +++ b/src/lib-dcrypt/dcrypt-openssl.c @@ -0,0 +1,2164 @@ +#include "lib.h" +#include "buffer.h" +#include "str.h" +#include "hex-binary.h" +#include "safe-memset.h" +#include "randgen.h" +#include "array.h" +#include "module-dir.h" +#include "dovecot-openssl-common.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "dcrypt.h" +#include "dcrypt-private.h" + +/** + + key format documentation: + ========================= + + v1 key + ------ + algo id = openssl NID + enctype = 0 = none, 1 = ecdhe, 2 = password + key id = sha256(hex encoded public point) + + public key + ---------- + 1algo idpublic point + + private key + ----------- + - enctype none + 1algo id0private pointkey id + + - enctype ecdh (algorithm AES-256-CTR, key = SHA256(shared secret), IV = \0\0\0...) + 1algo id1private pointephemeral public keyencryption key idkey id + + - enctype password (algorithm AES-256-CTR, key = PBKDF2(SHA1, 16, password, salt), IV = \0\0\0...) + 1algo id2private pointsaltkey id + + v2 key + ------ + algo oid = ASN1 OID of key algorithm (RSA or EC curve) + enctype = 0 = none, 1 = ecdhe, 2 = password + key id = SHA256(i2d_PUBKEY) + + public key + ---------- + 2HEX(i2d_PUBKEY)key id + + - enctype none + 2key algo oid0(RSA = i2d_PrivateKey, EC=Private Point)key id + + - enctype ecdh, key,iv = PBKDF2(hash algo, rounds, shared secret, salt) + 2key algo oid1symmetric algo namesalthash algoroundsE(RSA = i2d_PrivateKey, EC=Private Point)ephemeral public keyencryption key idkey id + + - enctype password, key,iv = PBKDF2(hash algo, rounds, password, salt) + 2key algo oid1symmetric algo namesalthash algoroundsE(RSA = i2d_PrivateKey, EC=Private Point)key id +**/ + +#if SSLEAY_VERSION_NUMBER < 0x1010000fL +#define EVP_PKEY_get0_EC_KEY(x) x->pkey.ec +#define EVP_PKEY_get0_RSA(x) x->pkey.rsa +#endif + +struct dcrypt_context_symmetric { + pool_t pool; + const EVP_CIPHER *cipher; + EVP_CIPHER_CTX *ctx; + unsigned char *key; + unsigned char *iv; + unsigned char *aad; + size_t aad_len; + unsigned char *tag; + size_t tag_len; + int padding; + int mode; +}; + +struct dcrypt_context_hmac { + pool_t pool; + const EVP_MD *md; +#if SSLEAY_VERSION_NUMBER >= 0x1010000fL + HMAC_CTX *ctx; +#else + HMAC_CTX ctx; +#endif + unsigned char *key; + size_t klen; +}; + +struct dcrypt_public_key { + void *ctx; +}; + +struct dcrypt_private_key { + void *ctx; +}; + +static +bool dcrypt_openssl_public_key_id(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r); +static +bool dcrypt_openssl_public_key_id_old(struct dcrypt_public_key *key, buffer_t *result, const char **error_r); +static +bool dcrypt_openssl_private_key_id(struct dcrypt_private_key *key, const char *algorithm, buffer_t *result, const char **error_r); +static +bool dcrypt_openssl_private_key_id_old(struct dcrypt_private_key *key, buffer_t *result, const char **error_r); +static +void dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r); +static +void dcrypt_openssl_free_private_key(struct dcrypt_private_key **key); +static +void dcrypt_openssl_free_public_key(struct dcrypt_public_key **key); +static +bool dcrypt_openssl_rsa_decrypt(struct dcrypt_private_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r); + +static +bool dcrypt_openssl_error(const char **error_r) +{ + if(error_r == NULL) return FALSE; /* caller is not really interested */ + unsigned long ec = ERR_get_error(); + *error_r = t_strdup_printf("%s", ERR_error_string(ec, NULL)); + return FALSE; +} + +static +bool dcrypt_openssl_initialize(const struct dcrypt_settings *set, const char **error_r) +{ + if (set->crypto_device != NULL && set->crypto_device[0] != '\0') { + if (dovecot_openssl_common_global_set_engine(set->crypto_device, error_r) <= 0) + return FALSE; + } + return TRUE; +} + +/* legacy function for old formats that generates + hex encoded point from EC public key + */ +static +char *ec_key_get_pub_point_hex(const EC_KEY *key) +{ + const EC_POINT *p; + const EC_GROUP *g; + + p = EC_KEY_get0_public_key(key); + g = EC_KEY_get0_group(key); + return EC_POINT_point2hex(g, p, POINT_CONVERSION_COMPRESSED, NULL); +} + +static +bool dcrypt_openssl_ctx_sym_create(const char *algorithm, enum dcrypt_sym_mode mode, struct dcrypt_context_symmetric **ctx_r, const char **error_r) +{ + struct dcrypt_context_symmetric *ctx; + pool_t pool; + const EVP_CIPHER *cipher; + cipher = EVP_get_cipherbyname(algorithm); + if (cipher == NULL) { + if (error_r != NULL) + *error_r = t_strdup_printf("Invalid cipher %s", algorithm); + return FALSE; + } + /* allocate context */ + pool = pool_alloconly_create("dcrypt openssl", 1024); + ctx = p_new(pool, struct dcrypt_context_symmetric, 1); + ctx->pool = pool; + ctx->cipher = cipher; + ctx->padding = 1; + ctx->mode =( mode == DCRYPT_MODE_ENCRYPT ? 1 : 0 ); + *ctx_r = ctx; + return TRUE; +} + +static +void dcrypt_openssl_ctx_sym_destroy(struct dcrypt_context_symmetric **ctx) +{ + pool_t pool = (*ctx)->pool; + if ((*ctx)->ctx) EVP_CIPHER_CTX_free((*ctx)->ctx); + pool_unref(&pool); + *ctx = NULL; +} + +static +void dcrypt_openssl_ctx_sym_set_key(struct dcrypt_context_symmetric *ctx, const unsigned char *key, size_t key_len) +{ + if(ctx->key != NULL) p_free(ctx->pool, ctx->key); + ctx->key = p_malloc(ctx->pool, EVP_CIPHER_key_length(ctx->cipher)); + memcpy(ctx->key, key, I_MIN(key_len,(size_t)EVP_CIPHER_key_length(ctx->cipher))); +} + +static +void dcrypt_openssl_ctx_sym_set_iv(struct dcrypt_context_symmetric *ctx, const unsigned char *iv, size_t iv_len) +{ + if(ctx->iv != NULL) p_free(ctx->pool, ctx->iv); + ctx->iv = p_malloc(ctx->pool, EVP_CIPHER_iv_length(ctx->cipher)); + memcpy(ctx->iv, iv, I_MIN(iv_len,(size_t)EVP_CIPHER_iv_length(ctx->cipher))); +} + +static +void dcrypt_openssl_ctx_sym_set_key_iv_random(struct dcrypt_context_symmetric *ctx) +{ + if(ctx->key != NULL) p_free(ctx->pool, ctx->key); + if(ctx->iv != NULL) p_free(ctx->pool, ctx->iv); + ctx->key = p_malloc(ctx->pool, EVP_CIPHER_key_length(ctx->cipher)); + random_fill(ctx->key, EVP_CIPHER_key_length(ctx->cipher)); + ctx->iv = p_malloc(ctx->pool, EVP_CIPHER_iv_length(ctx->cipher)); + random_fill(ctx->iv, EVP_CIPHER_iv_length(ctx->cipher)); +} + +static +void dcrypt_openssl_ctx_sym_set_padding(struct dcrypt_context_symmetric *ctx, bool padding) +{ + ctx->padding = (padding?1:0); +} + +static +bool dcrypt_openssl_ctx_sym_get_key(struct dcrypt_context_symmetric *ctx, buffer_t *key) +{ + if(ctx->key == NULL) return FALSE; + buffer_append(key, ctx->key, EVP_CIPHER_key_length(ctx->cipher)); + return TRUE; +} +static +bool dcrypt_openssl_ctx_sym_get_iv(struct dcrypt_context_symmetric *ctx, buffer_t *iv) +{ + if(ctx->iv == NULL) return FALSE; + buffer_append(iv, ctx->iv, EVP_CIPHER_iv_length(ctx->cipher)); + return TRUE; +} + +static +void dcrypt_openssl_ctx_sym_set_aad(struct dcrypt_context_symmetric *ctx, const unsigned char *aad, size_t aad_len) +{ + if (ctx->aad != NULL) p_free(ctx->pool, ctx->aad); + /* allow empty aad */ + ctx->aad = p_malloc(ctx->pool, I_MAX(1,aad_len)); + memcpy(ctx->aad, aad, aad_len); + ctx->aad_len = aad_len; +} + +static +bool dcrypt_openssl_ctx_sym_get_aad(struct dcrypt_context_symmetric *ctx, buffer_t *aad) +{ + if (ctx->aad == NULL) return FALSE; + buffer_append(aad, ctx->aad, ctx->aad_len); + return TRUE; +} + +static +void dcrypt_openssl_ctx_sym_set_tag(struct dcrypt_context_symmetric *ctx, const unsigned char *tag, size_t tag_len) +{ + if (ctx->tag != NULL) p_free(ctx->pool, ctx->tag); + /* unlike aad, tag cannot be empty */ + ctx->tag = p_malloc(ctx->pool, tag_len); + memcpy(ctx->tag, tag, tag_len); + ctx->tag_len = tag_len; +} + +static +bool dcrypt_openssl_ctx_sym_get_tag(struct dcrypt_context_symmetric *ctx, buffer_t *tag) +{ + if (ctx->tag == NULL) return FALSE; + buffer_append(tag, ctx->tag, ctx->tag_len); + return TRUE; +} + +static +unsigned int dcrypt_openssl_ctx_sym_get_key_length(struct dcrypt_context_symmetric *ctx) +{ + return EVP_CIPHER_iv_length(ctx->cipher); +} +static +unsigned int dcrypt_openssl_ctx_sym_get_iv_length(struct dcrypt_context_symmetric *ctx) +{ + return EVP_CIPHER_iv_length(ctx->cipher); +} +static +unsigned int dcrypt_openssl_ctx_sym_get_block_size(struct dcrypt_context_symmetric *ctx) +{ + return EVP_CIPHER_block_size(ctx->cipher); +} + +static +bool dcrypt_openssl_ctx_sym_init(struct dcrypt_context_symmetric *ctx, const char **error_r) +{ + int ec; + int len; + i_assert(ctx->key != NULL); + i_assert(ctx->iv != NULL); + i_assert(ctx->ctx == NULL); + + if((ctx->ctx = EVP_CIPHER_CTX_new()) == NULL) + return dcrypt_openssl_error(error_r); + + ec = EVP_CipherInit_ex(ctx->ctx, ctx->cipher, NULL, ctx->key, ctx->iv, ctx->mode); + if (ec != 1) return dcrypt_openssl_error(error_r); + EVP_CIPHER_CTX_set_padding(ctx->ctx, ctx->padding); + len = 0; + if (ctx->aad != NULL) ec = EVP_CipherUpdate(ctx->ctx, NULL, &len, ctx->aad, ctx->aad_len); + if (ec != 1) return dcrypt_openssl_error(error_r); + return TRUE; +} + +static +bool dcrypt_openssl_ctx_sym_update(struct dcrypt_context_symmetric *ctx, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r) +{ + const size_t block_size = (size_t)EVP_CIPHER_block_size(ctx->cipher); + size_t buf_used = result->used; + unsigned char *buf; + int outl; + + i_assert(ctx->ctx != NULL); + + /* From `man 3 evp_cipherupdate`: + + EVP_EncryptUpdate() encrypts inl bytes from the buffer in and writes + the encrypted version to out. This function can be called multiple + times to encrypt successive blocks of data. The amount of data written + depends on the block alignment of the encrypted data: as a result the + amount of data written may be anything from zero bytes to + (inl + cipher_block_size - 1) so out should contain sufficient room. + The actual number of bytes written is placed in outl. + */ + + buf = buffer_append_space_unsafe(result, data_len + block_size); + outl = 0; + if (EVP_CipherUpdate + (ctx->ctx, buf, &outl, data, data_len) != 1) + return dcrypt_openssl_error(error_r); + buffer_set_used_size(result, buf_used + outl); + return TRUE; +} + +static +bool dcrypt_openssl_ctx_sym_final(struct dcrypt_context_symmetric *ctx, buffer_t *result, const char **error_r) +{ + const size_t block_size = (size_t)EVP_CIPHER_block_size(ctx->cipher); + size_t buf_used = result->used; + unsigned char *buf; + int outl; + int ec; + + i_assert(ctx->ctx != NULL); + + /* From `man 3 evp_cipherupdate`: + + If padding is enabled (the default) then EVP_EncryptFinal_ex() encrypts + the "final" data, that is any data that remains in a partial block. It + uses standard block padding (aka PKCS padding). The encrypted final data + is written to out which should have sufficient space for one cipher + block. The number of bytes written is placed in outl. After this + function is called the encryption operation is finished and no further + calls to EVP_EncryptUpdate() should be made. + */ + + buf = buffer_append_space_unsafe(result, block_size); + outl = 0; + + /* when **DECRYPTING** set expected tag */ + if (ctx->mode == 0 && ctx->tag != NULL) { + ec = EVP_CIPHER_CTX_ctrl(ctx->ctx, EVP_CTRL_GCM_SET_TAG, ctx->tag_len, ctx->tag); + } else ec = 1; + + if (ec == 1) + ec = EVP_CipherFinal_ex(ctx->ctx, buf, &outl); + + if (ec == 1) { + buffer_set_used_size(result, buf_used + outl); + /* when **ENCRYPTING** recover tag */ + if (ctx->mode == 1 && ctx->aad != NULL) { + /* tag should be NULL here */ + i_assert(ctx->tag == NULL); + /* openssl claims taglen is always 16, go figure .. */ + ctx->tag = p_malloc(ctx->pool, EVP_GCM_TLS_TAG_LEN); + ec = EVP_CIPHER_CTX_ctrl(ctx->ctx, EVP_CTRL_GCM_GET_TAG, EVP_GCM_TLS_TAG_LEN, ctx->tag); + ctx->tag_len = EVP_GCM_TLS_TAG_LEN; + } + } + + if (ec == 0 && error_r != NULL) + *error_r = "data authentication failed"; + else if (ec < 0) dcrypt_openssl_error(error_r); + + EVP_CIPHER_CTX_free(ctx->ctx); + ctx->ctx = NULL; + + return ec == 1; +} + +static +bool dcrypt_openssl_ctx_hmac_create(const char *algorithm, struct dcrypt_context_hmac **ctx_r, const char **error_r) +{ + struct dcrypt_context_hmac *ctx; + pool_t pool; + const EVP_MD *md; + md = EVP_get_digestbyname(algorithm); + if(md == NULL) { + if (error_r != NULL) + *error_r = t_strdup_printf("Invalid digest %s", algorithm); + return FALSE; + } + /* allocate context */ + pool = pool_alloconly_create("dcrypt openssl", 1024); + ctx = p_new(pool, struct dcrypt_context_hmac, 1); + ctx->pool = pool; + ctx->md = md; + *ctx_r = ctx; + return TRUE; +} + +static +void dcrypt_openssl_ctx_hmac_destroy(struct dcrypt_context_hmac **ctx) +{ + pool_t pool = (*ctx)->pool; +#if SSLEAY_VERSION_NUMBER >= 0x1010000fL + if ((*ctx)->ctx) HMAC_CTX_free((*ctx)->ctx); +#else + HMAC_cleanup(&((*ctx)->ctx)); +#endif + pool_unref(&pool); + *ctx = NULL; +} + +static +void dcrypt_openssl_ctx_hmac_set_key(struct dcrypt_context_hmac *ctx, const unsigned char *key, size_t key_len) +{ + if(ctx->key != NULL) p_free(ctx->pool, ctx->key); + ctx->klen = I_MIN(key_len, HMAC_MAX_MD_CBLOCK); + ctx->key = p_malloc(ctx->pool, ctx->klen); + memcpy(ctx->key, key, ctx->klen); +} +static +bool dcrypt_openssl_ctx_hmac_get_key(struct dcrypt_context_hmac *ctx, buffer_t *key) +{ + if(ctx->key == NULL) return FALSE; + buffer_append(key, ctx->key, ctx->klen); + return TRUE; +} +static +void dcrypt_openssl_ctx_hmac_set_key_random(struct dcrypt_context_hmac *ctx) +{ + ctx->klen = HMAC_MAX_MD_CBLOCK; + ctx->key = p_malloc(ctx->pool, ctx->klen); + random_fill(ctx->key, ctx->klen); +} + +static +unsigned int dcrypt_openssl_ctx_hmac_get_digest_length(struct dcrypt_context_hmac *ctx) +{ + return EVP_MD_size(ctx->md); +} + +static +bool dcrypt_openssl_ctx_hmac_init(struct dcrypt_context_hmac *ctx, const char **error_r) +{ + int ec; + i_assert(ctx->md != NULL); +#if SSLEAY_VERSION_NUMBER >= 0x1010000fL + ctx->ctx = HMAC_CTX_new(); + if (ctx->ctx == NULL) return dcrypt_openssl_error(error_r); + ec = HMAC_Init_ex(ctx->ctx, ctx->key, ctx->klen, ctx->md, NULL); +#else + HMAC_CTX_init(&ctx->ctx); + ec = HMAC_Init_ex(&(ctx->ctx), ctx->key, ctx->klen, ctx->md, NULL); +#endif + if (ec != 1) return dcrypt_openssl_error(error_r); + return TRUE; +} +static +bool dcrypt_openssl_ctx_hmac_update(struct dcrypt_context_hmac *ctx, const unsigned char *data, size_t data_len, const char **error_r) +{ + int ec; +#if SSLEAY_VERSION_NUMBER >= 0x1010000fL + ec = HMAC_Update(ctx->ctx, data, data_len); +#else + ec = HMAC_Update(&(ctx->ctx), data, data_len); +#endif + if (ec != 1) return dcrypt_openssl_error(error_r); + return TRUE; +} +static +bool dcrypt_openssl_ctx_hmac_final(struct dcrypt_context_hmac *ctx, buffer_t *result, const char **error_r) +{ + int ec; + unsigned char buf[HMAC_MAX_MD_CBLOCK]; + unsigned int outl; +#if SSLEAY_VERSION_NUMBER >= 0x1010000fL + ec = HMAC_Final(ctx->ctx, buf, &outl); + HMAC_CTX_free(ctx->ctx); + ctx->ctx = NULL; +#else + ec = HMAC_Final(&(ctx->ctx), buf, &outl); + HMAC_cleanup(&(ctx->ctx)); +#endif + if (ec == 1) { + buffer_append(result, buf, outl); + } else return dcrypt_openssl_error(error_r); + return TRUE; +} + +static +bool dcrypt_openssl_generate_ec_key(int nid, EVP_PKEY **key, const char **error_r) +{ + EVP_PKEY_CTX *pctx; + EVP_PKEY_CTX *ctx; + EVP_PKEY *params = NULL; + + /* generate parameters for EC */ + pctx = EVP_PKEY_CTX_new_id(EVP_PKEY_EC, NULL); + if (pctx == NULL || + EVP_PKEY_paramgen_init(pctx) < 1 || + EVP_PKEY_CTX_set_ec_paramgen_curve_nid(pctx, nid) < 1 || + EVP_PKEY_paramgen(pctx, ¶ms) < 1) + { + dcrypt_openssl_error(error_r); + EVP_PKEY_CTX_free(pctx); + return FALSE; + } + + /* generate key from parameters */ + ctx = EVP_PKEY_CTX_new(params, NULL); + if (ctx == NULL || + EVP_PKEY_keygen_init(ctx) < 1 || + EVP_PKEY_keygen(ctx, key) < 1) + { + dcrypt_openssl_error(error_r); + EVP_PKEY_free(params); + EVP_PKEY_CTX_free(pctx); + EVP_PKEY_CTX_free(ctx); + return FALSE; + } + + EVP_PKEY_free(params); + EVP_PKEY_CTX_free(pctx); + EVP_PKEY_CTX_free(ctx); + EC_KEY_set_asn1_flag(EVP_PKEY_get0_EC_KEY((*key)), OPENSSL_EC_NAMED_CURVE); + EC_KEY_set_conv_form(EVP_PKEY_get0_EC_KEY((*key)), POINT_CONVERSION_COMPRESSED); + return TRUE; +} + +static +bool dcrypt_openssl_generate_rsa_key(int bits, EVP_PKEY **key, const char **error_r) +{ + i_assert(bits >= 256); + int ec = 0; + + EVP_PKEY_CTX *ctx; + ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL); + if (ctx == NULL || + EVP_PKEY_keygen_init(ctx) < 1 || + EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, bits) < 1 || + EVP_PKEY_keygen(ctx, key) < 1) { + dcrypt_openssl_error(error_r); + ec = -1; + } + + EVP_PKEY_CTX_free(ctx); + return ec == 0; +} + +static +bool dcrypt_openssl_ecdh_derive_secret_local(struct dcrypt_private_key *local_key, buffer_t *R, buffer_t *S, const char **error_r) +{ + i_assert(local_key != NULL); + EVP_PKEY *local = (EVP_PKEY*)local_key; + BN_CTX *bn_ctx = BN_CTX_new(); + if (bn_ctx == NULL) + return dcrypt_openssl_error(error_r); + const EC_GROUP *grp = EC_KEY_get0_group(EVP_PKEY_get0_EC_KEY(local)); + EC_POINT *pub = EC_POINT_new(grp); + /* convert ephemeral key data EC point */ + if (pub == NULL || + EC_POINT_oct2point(grp, pub, R->data, R->used, bn_ctx) != 1) + { + EC_POINT_free(pub); + BN_CTX_free(bn_ctx); + return dcrypt_openssl_error(error_r); + } + EC_KEY *ec_key = EC_KEY_new(); + /* convert point to public key */ + int ec = 0; + if (ec_key == NULL || + EC_KEY_set_group(ec_key, grp) != 1 || + EC_KEY_set_public_key(ec_key, pub) != 1) + ec = -1; + else + EC_KEY_set_conv_form(ec_key, POINT_CONVERSION_COMPRESSED); + EC_POINT_free(pub); + BN_CTX_free(bn_ctx); + + /* make sure it looks like a valid key */ + if (ec == -1 || EC_KEY_check_key(ec_key) != 1) { + EC_KEY_free(ec_key); + return dcrypt_openssl_error(error_r); + } + + EVP_PKEY *peer = EVP_PKEY_new(); + if (peer == NULL) { + EC_KEY_free(ec_key); + return dcrypt_openssl_error(error_r); + } + EVP_PKEY_set1_EC_KEY(peer, ec_key); + EC_KEY_free(ec_key); + EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new(local, NULL); + + /* initialize derivation */ + if (pctx == NULL || + EVP_PKEY_derive_init(pctx) != 1 || + EVP_PKEY_derive_set_peer(pctx, peer) != 1) { + EVP_PKEY_CTX_free(pctx); + EVP_PKEY_free(peer); + return dcrypt_openssl_error(error_r); + } + + /* have to do it twice to get the data length */ + size_t len; + if (EVP_PKEY_derive(pctx, NULL, &len) != 1) { + EVP_PKEY_CTX_free(pctx); + EVP_PKEY_free(peer); + return dcrypt_openssl_error(error_r); + } + unsigned char buf[len]; + memset(buf,0,len); + if (EVP_PKEY_derive(pctx, buf, &len) != 1) { + EVP_PKEY_CTX_free(pctx); + EVP_PKEY_free(peer); + return dcrypt_openssl_error(error_r); + } + EVP_PKEY_CTX_free(pctx); + buffer_append(S, buf, len); + EVP_PKEY_free(peer); + return TRUE; +} + +static +bool dcrypt_openssl_ecdh_derive_secret_peer(struct dcrypt_public_key *peer_key, buffer_t *R, buffer_t *S, const char **error_r) +{ + /* ensure peer_key is EC key */ + EVP_PKEY *local = NULL; + EVP_PKEY *peer = (EVP_PKEY*)peer_key; + if (EVP_PKEY_base_id(peer) != EVP_PKEY_EC) { + if (error_r != NULL) + *error_r = "Only ECC key can be used"; + return FALSE; + } + + /* generate another key from same group */ + int nid = EC_GROUP_get_curve_name(EC_KEY_get0_group(EVP_PKEY_get0_EC_KEY(peer))); + if (!dcrypt_openssl_generate_ec_key(nid, &local, error_r)) return FALSE; + + /* initialize */ + EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new(local, NULL); + if (pctx == NULL || + EVP_PKEY_derive_init(pctx) != 1 || + EVP_PKEY_derive_set_peer(pctx, peer) != 1) { + EVP_PKEY_CTX_free(pctx); + return dcrypt_openssl_error(error_r); + } + + /* derive */ + size_t len; + if (EVP_PKEY_derive(pctx, NULL, &len) != 1) { + EVP_PKEY_CTX_free(pctx); + return dcrypt_openssl_error(error_r); + } + unsigned char buf[len]; + if (EVP_PKEY_derive(pctx, buf, &len) != 1) { + EVP_PKEY_CTX_free(pctx); + return dcrypt_openssl_error(error_r); + } + + EVP_PKEY_CTX_free(pctx); + buffer_append(S, buf, len); + + /* get ephemeral key (=R) */ + BN_CTX *bn_ctx = BN_CTX_new(); + const EC_POINT *pub = EC_KEY_get0_public_key(EVP_PKEY_get0_EC_KEY(local)); + const EC_GROUP *grp = EC_KEY_get0_group(EVP_PKEY_get0_EC_KEY(local)); + len = EC_POINT_point2oct(grp, pub, POINT_CONVERSION_COMPRESSED, NULL, 0, bn_ctx); + unsigned char R_buf[len]; + EC_POINT_point2oct(grp, pub, POINT_CONVERSION_COMPRESSED, R_buf, len, bn_ctx); + BN_CTX_free(bn_ctx); + buffer_append(R, R_buf, len); + EVP_PKEY_free(local); + + return TRUE; +} + +static +bool dcrypt_openssl_pbkdf2(const unsigned char *password, size_t password_len, const unsigned char *salt, size_t salt_len, + const char *hash, unsigned int rounds, buffer_t *result, unsigned int result_len, const char **error_r) +{ + int ret; + i_assert(rounds > 0); + i_assert(result_len > 0); + i_assert(result != NULL); + /* determine MD */ + const EVP_MD* md = EVP_get_digestbyname(hash); + if (md == NULL) { + if (error_r != NULL) + *error_r = t_strdup_printf("Invalid digest %s", hash); + return FALSE; + } + + unsigned char buffer[result_len]; + if ((ret = PKCS5_PBKDF2_HMAC((const char*)password, password_len, salt, salt_len, rounds, + md, result_len, buffer)) == 1) { + buffer_append(result, buffer, result_len); + } + if (ret != 1) return dcrypt_openssl_error(error_r); + return TRUE; +} + +static +bool dcrypt_openssl_generate_keypair(struct dcrypt_keypair *pair_r, enum dcrypt_key_type kind, unsigned int bits, const char *curve, const char **error_r) +{ + EVP_PKEY *pkey = NULL; + if (kind == DCRYPT_KEY_RSA) { + if (dcrypt_openssl_generate_rsa_key(bits, &pkey, error_r)) { + pair_r->priv = (struct dcrypt_private_key*)pkey; + dcrypt_openssl_private_to_public_key(pair_r->priv, &(pair_r->pub)); + return TRUE; + } else return dcrypt_openssl_error(error_r); + } else if (kind == DCRYPT_KEY_EC) { + int nid = OBJ_sn2nid(curve); + if (nid == NID_undef) { + if (error_r != NULL) + *error_r = t_strdup_printf("Unknown EC curve %s", curve); + return FALSE; + } + if (dcrypt_openssl_generate_ec_key(nid, &pkey, error_r)) { + pair_r->priv = (struct dcrypt_private_key*)pkey; + dcrypt_openssl_private_to_public_key(pair_r->priv, &(pair_r->pub)); + return TRUE; + } else return dcrypt_openssl_error(error_r); + } + if (error_r != NULL) + *error_r = "Key type not supported in this build"; + return FALSE; +} + +static +bool dcrypt_openssl_decrypt_point_v1(buffer_t *data, buffer_t *key, BIGNUM **point_r, const char **error_r) +{ + struct dcrypt_context_symmetric *dctx; + buffer_t *tmp = buffer_create_dynamic(pool_datastack_create(), 64); + + if (!dcrypt_openssl_ctx_sym_create("aes-256-ctr", DCRYPT_MODE_DECRYPT, &dctx, error_r)) { + return FALSE; + } + + /* v1 KEYS have all-zero IV - have to use it ourselves too */ + dcrypt_openssl_ctx_sym_set_iv(dctx, (const unsigned char*)"\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0", 16); + dcrypt_openssl_ctx_sym_set_key(dctx, key->data, key->used); + + if (!dcrypt_openssl_ctx_sym_init(dctx, error_r) || + !dcrypt_openssl_ctx_sym_update(dctx, data->data, data->used, tmp, error_r) || + !dcrypt_openssl_ctx_sym_final(dctx, tmp, error_r)) { + dcrypt_openssl_ctx_sym_destroy(&dctx); + return FALSE; + } + + dcrypt_openssl_ctx_sym_destroy(&dctx); + + *point_r = BN_bin2bn(tmp->data, tmp->used, NULL); + safe_memset(buffer_get_modifiable_data(tmp, NULL), 0,tmp->used); + buffer_set_used_size(key, 0); + + if (*point_r == FALSE) + return dcrypt_openssl_error(error_r); + + return TRUE; +} + +static +bool dcrypt_openssl_decrypt_point_ec_v1(struct dcrypt_private_key *dec_key, + const char *data_hex, const char *peer_key_hex, BIGNUM **point_r, const char **error_r) +{ + buffer_t *peer_key, *data, key, *secret; + bool res; + + data = buffer_create_dynamic(pool_datastack_create(), 128); + peer_key = buffer_create_dynamic(pool_datastack_create(), 64); + + hex_to_binary(data_hex, data); + hex_to_binary(peer_key_hex, peer_key); + + secret = buffer_create_dynamic(pool_datastack_create(), 64); + + if (!dcrypt_openssl_ecdh_derive_secret_local(dec_key, peer_key, secret, error_r)) + return FALSE; + + /* run it thru SHA256 once */ + unsigned char digest[SHA256_DIGEST_LENGTH]; + SHA256(secret->data, secret->used, digest); + safe_memset(buffer_get_modifiable_data(secret, NULL), 0, secret->used); + buffer_set_used_size(secret, 0); + buffer_create_from_const_data(&key, digest, SHA256_DIGEST_LENGTH); + + /* then use this as key */ + res = dcrypt_openssl_decrypt_point_v1(data, &key, point_r, error_r); + memset(digest, 0, sizeof(digest)); + safe_memset(digest, 0, SHA256_DIGEST_LENGTH); + + return res; +} + +static +bool dcrypt_openssl_decrypt_point_password_v1(const char *data_hex, const char *password_hex, + const char *salt_hex, BIGNUM **point_r, const char **error_r) +{ + buffer_t *salt, *data, *password, *key; + struct dcrypt_context_symmetric *dctx; + + data = buffer_create_dynamic(pool_datastack_create(), 128); + salt = buffer_create_dynamic(pool_datastack_create(), 16); + password = buffer_create_dynamic(pool_datastack_create(), 32); + key = buffer_create_dynamic(pool_datastack_create(), 32); + + hex_to_binary(data_hex, data); + hex_to_binary(salt_hex, salt); + hex_to_binary(password_hex, password); + + /* aes-256-ctr uses 32 byte key, and v1 uses all-zero IV */ + if (!dcrypt_openssl_pbkdf2(password->data, password->used, salt->data, salt->used, + "sha256", 16, key, 32, error_r)) { + dcrypt_ctx_sym_destroy(&dctx); + return FALSE; + } + + return dcrypt_openssl_decrypt_point_v1(data, key, point_r, error_r); +} + +static +bool dcrypt_openssl_load_private_key_dovecot_v1(struct dcrypt_private_key **key_r, + int len, const char **input, const char *password, struct dcrypt_private_key *dec_key, + const char **error_r) +{ + int nid, ec, enctype; + BIGNUM *point = NULL; + + if (str_to_int(input[1], &nid) != 0) { + if (error_r != NULL) + *error_r = "Corrupted data"; + return FALSE; + } + + if (str_to_int(input[2], &enctype) != 0) { + if (error_r != NULL) + *error_r = "Corrupted data"; + return FALSE; + } + + /* decode and optionally decipher private key value */ + if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_NONE) { + point = BN_new(); + if (point == NULL || BN_hex2bn(&point, input[3]) < 1) { + BN_free(point); + return dcrypt_openssl_error(error_r); + } + } else if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PASSWORD) { + /* by password */ + const char *enc_priv_pt = input[3]; + const char *salt = input[4]; + if (!dcrypt_openssl_decrypt_point_password_v1(enc_priv_pt, password, salt, &point, error_r)) { + return FALSE; + } + } else if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PK) { + /* by key */ + const char *enc_priv_pt = input[3]; + const char *peer_key = input[4]; + i_assert(dec_key != NULL); + if (!dcrypt_openssl_decrypt_point_ec_v1(dec_key, enc_priv_pt, peer_key, &point, error_r)) { + return FALSE; + } + } else { + if (error_r != NULL) + *error_r = "Invalid key data"; + return FALSE; + } + + EC_KEY *eckey = EC_KEY_new_by_curve_name(nid); + if (eckey == NULL) return dcrypt_openssl_error(error_r); + + /* assign private key */ + BN_CTX *bnctx = BN_CTX_new(); + if (bnctx == NULL) { + EC_KEY_free(eckey); + return dcrypt_openssl_error(error_r); + } + EC_KEY_set_conv_form(eckey, POINT_CONVERSION_COMPRESSED); + EC_KEY_set_private_key(eckey, point); + EC_KEY_precompute_mult(eckey, bnctx); + EC_KEY_set_asn1_flag(eckey, OPENSSL_EC_NAMED_CURVE); + EC_POINT *pub = EC_POINT_new(EC_KEY_get0_group(eckey)); + if (pub == NULL) { + EC_KEY_free(eckey); + BN_CTX_free(bnctx); + return dcrypt_openssl_error(error_r); + } + /* calculate public key */ + ec = EC_POINT_mul(EC_KEY_get0_group(eckey), pub, point, NULL, NULL, bnctx); + EC_KEY_set_public_key(eckey, pub); + BN_free(point); + EC_POINT_free(pub); + BN_CTX_free(bnctx); + + /* make sure it looks OK and is correct */ + if (ec == 1 && EC_KEY_check_key(eckey) == 1) { + unsigned char digest[SHA256_DIGEST_LENGTH]; + /* validate that the key was loaded correctly */ + char *id = ec_key_get_pub_point_hex(eckey); + if (id == NULL) { + EC_KEY_free(eckey); + return dcrypt_openssl_error(error_r); + } + SHA256((unsigned char*)id, strlen(id), digest); + OPENSSL_free(id); + const char *digest_hex = binary_to_hex(digest, SHA256_DIGEST_LENGTH); + if (strcmp(digest_hex, input[len-1]) != 0) { + if (error_r != NULL) + *error_r = "Key id mismatch after load"; + EC_KEY_free(eckey); + return FALSE; + } + EVP_PKEY *key = EVP_PKEY_new(); + if (key == NULL) { + EC_KEY_free(eckey); + return dcrypt_openssl_error(error_r); + } + EVP_PKEY_set1_EC_KEY(key, eckey); + EC_KEY_free(eckey); + *key_r = (struct dcrypt_private_key *)key; + return TRUE; + } + + EC_KEY_free(eckey); + + return dcrypt_openssl_error(error_r); +} + +/* encrypt/decrypt private keys */ +static +bool dcrypt_openssl_cipher_key_dovecot_v2(const char *cipher, enum dcrypt_sym_mode mode, + buffer_t *input, buffer_t *secret, buffer_t *salt, const char *digalgo, unsigned int rounds, + buffer_t *result_r, const char **error_r) +{ + struct dcrypt_context_symmetric *dctx; + bool res; + + if (!dcrypt_openssl_ctx_sym_create(cipher, mode, &dctx, error_r)) { + return FALSE; + } + + /* generate encryption key/iv based on secret/salt */ + buffer_t *key_data = buffer_create_dynamic(pool_datastack_create(), 128); + res = dcrypt_openssl_pbkdf2(secret->data, secret->used, salt->data, salt->used, + digalgo, rounds, key_data, + dcrypt_openssl_ctx_sym_get_key_length(dctx)+dcrypt_openssl_ctx_sym_get_iv_length(dctx), error_r); + + if (!res) { + dcrypt_openssl_ctx_sym_destroy(&dctx); + return FALSE; + } + + buffer_t *tmp = buffer_create_dynamic(pool_datastack_create(), 128); + const unsigned char *kd = buffer_free_without_data(&key_data); + + /* perform ciphering */ + dcrypt_openssl_ctx_sym_set_key(dctx, kd, dcrypt_openssl_ctx_sym_get_key_length(dctx)); + dcrypt_openssl_ctx_sym_set_iv(dctx, kd+dcrypt_openssl_ctx_sym_get_key_length(dctx), dcrypt_openssl_ctx_sym_get_iv_length(dctx)); + + if (!dcrypt_openssl_ctx_sym_init(dctx, error_r) || + !dcrypt_openssl_ctx_sym_update(dctx, input->data, input->used, tmp, error_r) || + !dcrypt_openssl_ctx_sym_final(dctx, tmp, error_r)) { + res = FALSE; + } else { + /* provide result if succeeded */ + buffer_append_buf(result_r, tmp, 0, (size_t)-1); + res = TRUE; + } + /* and ensure no data leaks */ + safe_memset(buffer_get_modifiable_data(tmp, NULL), 0, tmp->used); + + dcrypt_openssl_ctx_sym_destroy(&dctx); + return res; +} + +static +bool dcrypt_openssl_load_private_key_dovecot_v2(struct dcrypt_private_key **key_r, + int len, const char **input, const char *password, struct dcrypt_private_key *dec_key, + const char **error_r) +{ + int enctype; + buffer_t *key_data = buffer_create_dynamic(pool_datastack_create(), 256); + + /* check for encryption type */ + if (str_to_int(input[2], &enctype) != 0) { + if (error_r != NULL) + *error_r = "Corrupted data"; + return FALSE; + } + + if (enctype < 0 || enctype > 2) { + if (error_r != NULL) + *error_r = "Corrupted data"; + return FALSE; + } + + /* match encryption type to field counts */ + if ((enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_NONE && len != 5) || + (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PASSWORD && len != 9) || + (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PK && len != 11)) { + if (error_r != NULL) + *error_r = "Corrupted data"; + return FALSE; + } + + /* get key type */ + int nid = OBJ_txt2nid(input[1]); + + if (nid == NID_undef) + return dcrypt_openssl_error(error_r); + + /* decode and possibly decipher private key value */ + if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_NONE) { + if (hex_to_binary(input[3], key_data) != 0) { + if (error_r != NULL) + *error_r = "Corrupted data"; + } + } else if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PK) { + unsigned int rounds; + struct dcrypt_public_key *pubkey = NULL; + if (str_to_uint(input[6], &rounds) != 0) { + if (error_r != NULL) + *error_r = "Corrupted data"; + return FALSE; + } + + buffer_t *data = buffer_create_dynamic(pool_datastack_create(), 128); + + /* check that we have correct decryption key */ + dcrypt_openssl_private_to_public_key(dec_key, &pubkey); + if (!dcrypt_openssl_public_key_id(pubkey, "sha256", data, error_r)) { + if (pubkey != NULL) dcrypt_openssl_free_public_key(&pubkey); + return FALSE; + } + + dcrypt_openssl_free_public_key(&pubkey); + + if (strcmp(binary_to_hex(data->data, data->used), input[9]) != 0) { + dcrypt_openssl_free_public_key(&pubkey); + if (error_r != NULL) + *error_r = "No private key available"; + return FALSE; + } + + + buffer_t *salt, *peer_key, *secret; + salt = buffer_create_dynamic(pool_datastack_create(), strlen(input[4])/2); + peer_key = buffer_create_dynamic(pool_datastack_create(), strlen(input[8])/2); + secret = buffer_create_dynamic(pool_datastack_create(), 128); + + buffer_set_used_size(data, 0); + hex_to_binary(input[4], salt); + hex_to_binary(input[8], peer_key); + hex_to_binary(input[7], data); + + /* get us secret value to use for key/iv generation */ + if (EVP_PKEY_base_id((EVP_PKEY*)dec_key) == EVP_PKEY_RSA) { + if (!dcrypt_openssl_rsa_decrypt(dec_key, peer_key->data, peer_key->used, secret, error_r)) + return FALSE; + } else { + /* perform ECDH */ + if (!dcrypt_openssl_ecdh_derive_secret_local(dec_key, peer_key, secret, error_r)) + return FALSE; + } + /* decrypt key */ + if (!dcrypt_openssl_cipher_key_dovecot_v2(input[3], DCRYPT_MODE_DECRYPT, data, secret, salt, + input[5], rounds, key_data, error_r)) { + return FALSE; + } + } else if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PASSWORD) { + unsigned int rounds; + if (str_to_uint(input[6], &rounds) != 0) { + if (error_r != NULL) + *error_r = "Corrupted data"; + return FALSE; + } + + buffer_t *salt, secret, *data; + salt = buffer_create_dynamic(pool_datastack_create(), strlen(input[4])/2); + buffer_create_from_const_data(&secret, password, strlen(password)); + data = buffer_create_dynamic(pool_datastack_create(), strlen(input[7])/2); + if (hex_to_binary(input[4], salt) != 0 || + hex_to_binary(input[7], data) != 0) { + if (error_r != NULL) + *error_r = "Corrupted data"; + return FALSE; + } + + if (!dcrypt_openssl_cipher_key_dovecot_v2(input[3], DCRYPT_MODE_DECRYPT, data, &secret, salt, + input[5], rounds, key_data, error_r)) { + return FALSE; + } + } + + /* decode actual key */ + if (EVP_PKEY_type(nid) == EVP_PKEY_RSA) { + RSA *rsa = RSA_new(); + const unsigned char *ptr = buffer_get_data(key_data, NULL); + if (rsa == NULL || + d2i_RSAPrivateKey(&rsa, &ptr, key_data->used) == NULL || + RSA_check_key(rsa) != 1) { + safe_memset(buffer_get_modifiable_data(key_data, NULL), 0, key_data->used); + RSA_free(rsa); + return dcrypt_openssl_error(error_r); + } + safe_memset(buffer_get_modifiable_data(key_data, NULL), 0, key_data->used); + buffer_set_used_size(key_data, 0); + EVP_PKEY *pkey = EVP_PKEY_new(); + if (pkey == NULL) { + RSA_free(rsa); + return dcrypt_openssl_error(error_r); + } + EVP_PKEY_set1_RSA(pkey, rsa); + RSA_free(rsa); + *key_r = (struct dcrypt_private_key *)pkey; + } else { + int ec; + BIGNUM *point = BN_new(); + if (point == NULL || + BN_mpi2bn(key_data->data, key_data->used, point) == NULL) { + safe_memset(buffer_get_modifiable_data(key_data, NULL), 0, key_data->used); + BN_free(point); + return dcrypt_openssl_error(error_r); + } + EC_KEY *eckey = EC_KEY_new_by_curve_name(nid); + safe_memset(buffer_get_modifiable_data(key_data, NULL), 0, key_data->used); + buffer_set_used_size(key_data, 0); + BN_CTX *bnctx = BN_CTX_new(); + if (eckey == NULL || bnctx == NULL) { + BN_free(point); + EC_KEY_free(eckey); + BN_CTX_free(bnctx); + return dcrypt_openssl_error(error_r); + } + EC_KEY_set_conv_form(eckey, POINT_CONVERSION_COMPRESSED); + EC_KEY_set_private_key(eckey, point); + EC_KEY_precompute_mult(eckey, bnctx); + EC_KEY_set_asn1_flag(eckey, OPENSSL_EC_NAMED_CURVE); + EC_POINT *pub = EC_POINT_new(EC_KEY_get0_group(eckey)); + if (pub == NULL) + ec = -1; + else { + /* calculate public key */ + ec = EC_POINT_mul(EC_KEY_get0_group(eckey), pub, point, NULL, NULL, bnctx); + EC_KEY_set_public_key(eckey, pub); + EC_POINT_free(pub); + } + BN_free(point); + BN_CTX_free(bnctx); + /* make sure the EC key is valid */ + EVP_PKEY *key = EVP_PKEY_new(); + if (ec == 1 && key != NULL && EC_KEY_check_key(eckey) == 1) { + EVP_PKEY_set1_EC_KEY(key, eckey); + EC_KEY_free(eckey); + *key_r = (struct dcrypt_private_key *)key; + } else { + EVP_PKEY_free(key); + EC_KEY_free(eckey); + return dcrypt_openssl_error(error_r); + } + } + + /* finally compare key to key id */ + dcrypt_openssl_private_key_id(*key_r, "sha256", key_data, NULL); + + if (strcmp(binary_to_hex(key_data->data, key_data->used), input[len-1]) != 0) { + dcrypt_openssl_free_private_key(key_r); + if (error_r != NULL) + *error_r = "Key id mismatch after load"; + return FALSE; + } + + return TRUE; +} + + +static +bool dcrypt_openssl_load_private_key_dovecot(struct dcrypt_private_key **key_r, + const char *data, const char *password, struct dcrypt_private_key *key, + const char **error_r) +{ + bool ret; + const char **input = t_strsplit_tab(data); + size_t len = str_array_length(input); + + if (len < 4) { + if (error_r != NULL) + *error_r = "Corrupted data"; + ret = FALSE; + } else if (*(input[0])== '1') + ret = dcrypt_openssl_load_private_key_dovecot_v1(key_r, len, input, password, key, error_r); + else if (*(input[0])== '2') + ret = dcrypt_openssl_load_private_key_dovecot_v2(key_r, len, input, password, key, error_r); + else { + if (error_r != NULL) + *error_r = "Unsupported key version"; + ret = FALSE; + } + return ret; +} + +static +int dcrypt_openssl_load_public_key_dovecot_v1(struct dcrypt_public_key **key_r, + int len, const char **input, const char **error_r) +{ + int nid; + if (len != 4) { + if (error_r != NULL) + *error_r = "Corrupted data"; + return -1; + } + if (str_to_int(input[1], &nid) != 0) { + if (error_r != NULL) + *error_r = "Corrupted data"; + return -1; + } + + EC_KEY *eckey = EC_KEY_new_by_curve_name(nid); + if (eckey == NULL) { + dcrypt_openssl_error(error_r); + return -1; + } + + EC_KEY_set_asn1_flag(eckey, OPENSSL_EC_NAMED_CURVE); + BN_CTX *bnctx = BN_CTX_new(); + + EC_POINT *point = EC_POINT_new(EC_KEY_get0_group(eckey)); + if (bnctx == NULL || point == NULL || + EC_POINT_hex2point(EC_KEY_get0_group(eckey), + input[2], point, bnctx) == NULL) { + BN_CTX_free(bnctx); + EC_KEY_free(eckey); + EC_POINT_free(point); + dcrypt_openssl_error(error_r); + return -1; + } + BN_CTX_free(bnctx); + + EC_KEY_set_public_key(eckey, point); + EC_KEY_set_asn1_flag(eckey, OPENSSL_EC_NAMED_CURVE); + + EC_POINT_free(point); + + if (EC_KEY_check_key(eckey) == 1) { + EVP_PKEY *key = EVP_PKEY_new(); + EVP_PKEY_set1_EC_KEY(key, eckey); + EC_KEY_free(eckey); + /* make sure digest matches */ + buffer_t *dgst = buffer_create_dynamic(pool_datastack_create(), 32); + dcrypt_openssl_public_key_id_old((struct dcrypt_public_key *)key, dgst, NULL); + if (strcmp(binary_to_hex(dgst->data, dgst->used), input[len-1]) != 0) { + if (error_r != NULL) + *error_r = "Key id mismatch after load"; + EVP_PKEY_free(key); + return -1; + } + *key_r = (struct dcrypt_public_key *)key; + return 0; + } + + dcrypt_openssl_error(error_r); + return -1; +} + +static +int dcrypt_openssl_load_public_key_dovecot_v2(struct dcrypt_public_key **key_r, + int len, const char **input, const char **error_r) +{ + if (len != 3 || strlen(input[1]) < 2 || (strlen(input[1])%2) != 0) { + if (error_r != NULL) + *error_r = "Corrupted data"; + return -1; + } + buffer_t tmp; + size_t keylen = strlen(input[1])/2; + unsigned char keybuf[keylen]; + const unsigned char *ptr; + buffer_create_from_data(&tmp, keybuf, keylen); + hex_to_binary(input[1], &tmp); + ptr = keybuf; + + EVP_PKEY *pkey = EVP_PKEY_new(); + if (pkey == NULL || d2i_PUBKEY(&pkey, &ptr, keylen)==NULL) { + EVP_PKEY_free(pkey); + dcrypt_openssl_error(error_r); + return -1; + } + + /* make sure digest matches */ + buffer_t *dgst = buffer_create_dynamic(pool_datastack_create(), 32); + dcrypt_openssl_public_key_id((struct dcrypt_public_key *)pkey, "sha256", dgst, NULL); + if (strcmp(binary_to_hex(dgst->data, dgst->used), input[len-1]) != 0) { + if (error_r != NULL) + *error_r = "Key id mismatch after load"; + EVP_PKEY_free(pkey); + return -1; + } + + *key_r = (struct dcrypt_public_key *)pkey; + return 0; +} + +static +bool dcrypt_openssl_load_public_key_dovecot(struct dcrypt_public_key **key_r, + const char *data, const char **error_r) +{ + int ec = 0; + + const char **input = t_strsplit_tab(data); + size_t len = str_array_length(input); + + if (len < 2) ec = -1; + if (ec == 0 && *(input[0]) == '1') { + ec = dcrypt_openssl_load_public_key_dovecot_v1(key_r, len, + input, error_r); + } else if (ec == 0 && *(input[0]) == '2') { + ec = dcrypt_openssl_load_public_key_dovecot_v2(key_r, len, + input, error_r); + } else { + if (error_r != NULL) + *error_r = "Unsupported key version"; + ec = -1; + } + + return (ec == 0 ? TRUE : FALSE); +} + +static +bool dcrypt_openssl_encrypt_private_key_dovecot(buffer_t *key, int enctype, const char *cipher, const char *password, + struct dcrypt_public_key *enc_key, buffer_t *destination, const char **error_r) +{ + bool res; + unsigned char *ptr; + + unsigned char salt[8]; + buffer_t *peer_key = buffer_create_dynamic(pool_datastack_create(), 128); + buffer_t *secret = buffer_create_dynamic(pool_datastack_create(), 128); + cipher = t_str_lcase(cipher); + + str_append(destination, cipher); + str_append_c(destination, '\t'); + random_fill(salt, sizeof(salt)); + binary_to_hex_append(destination, salt, sizeof(salt)); + buffer_t saltbuf; + buffer_create_from_const_data(&saltbuf, salt, sizeof(salt)); + + /* so we don't have to make new version if we ever upgrade these */ + str_append(destination, t_strdup_printf("\t%s\t%d\t", + DCRYPT_DOVECOT_KEY_ENCRYPT_HASH, + DCRYPT_DOVECOT_KEY_ENCRYPT_ROUNDS)); + + if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PK) { + if (EVP_PKEY_base_id((EVP_PKEY*)enc_key) == EVP_PKEY_RSA) { + size_t used = buffer_get_used_size(secret); + /* peer key, in this case, is encrypted secret, which is 16 bytes of data */ + ptr = buffer_append_space_unsafe(secret, 16); + random_fill(ptr, 16); + buffer_set_used_size(secret, used+16); + if (!dcrypt_rsa_encrypt(enc_key, secret->data, secret->used, peer_key, error_r)) { + return FALSE; + } + } else if (EVP_PKEY_base_id((EVP_PKEY*)enc_key) == EVP_PKEY_EC) { + /* generate secret by ECDHE */ + if (!dcrypt_openssl_ecdh_derive_secret_peer(enc_key, peer_key, secret, error_r)) { + return FALSE; + } + } else { + /* Loading the key should have failed */ + i_unreached(); + } + /* add encryption key id, reuse peer_key buffer */ + } else if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PASSWORD) { + str_append(secret, password); + } + + /* encrypt key using secret and salt */ + buffer_t *tmp = buffer_create_dynamic(pool_datastack_create(), 128); + res = dcrypt_openssl_cipher_key_dovecot_v2(cipher, DCRYPT_MODE_ENCRYPT, key, secret, &saltbuf, + DCRYPT_DOVECOT_KEY_ENCRYPT_HASH, DCRYPT_DOVECOT_KEY_ENCRYPT_ROUNDS, tmp, error_r); + safe_memset(buffer_get_modifiable_data(secret, NULL), 0, secret->used); + binary_to_hex_append(destination, tmp->data, tmp->used); + + /* some additional fields or private key version */ + if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PK) { + str_append_c(destination, '\t'); + + /* for RSA, this is the actual encrypted secret */ + binary_to_hex_append(destination, peer_key->data, peer_key->used); + str_append_c(destination, '\t'); + + buffer_set_used_size(peer_key, 0); + if (!dcrypt_openssl_public_key_id(enc_key, "sha256", peer_key, error_r)) + return FALSE; + binary_to_hex_append(destination, peer_key->data, peer_key->used); + } + return res; +} + +static +bool dcrypt_openssl_store_private_key_dovecot(struct dcrypt_private_key *key, const char *cipher, buffer_t *destination, + const char *password, struct dcrypt_public_key *enc_key, const char **error_r) +{ + size_t dest_used = buffer_get_used_size(destination); + const char *cipher2 = NULL; + EVP_PKEY *pkey = (EVP_PKEY*)key; + char objtxt[80]; /* openssl manual says this is OK */ + ASN1_OBJECT *obj; + if (EVP_PKEY_base_id(pkey) == EVP_PKEY_EC) { + /* because otherwise we get wrong nid */ + obj = OBJ_nid2obj(EC_GROUP_get_curve_name(EC_KEY_get0_group(EVP_PKEY_get0_EC_KEY(pkey)))); + + } else { + obj = OBJ_nid2obj(EVP_PKEY_id(pkey)); + } + + int enctype = 0; + int ln = OBJ_obj2txt(objtxt, sizeof(objtxt), obj, 1); + if (ln < 1) + return dcrypt_openssl_error(error_r); + if (ln > (int)sizeof(objtxt)) { + if (error_r != NULL) + *error_r = "Object identifier too long"; + return FALSE; + } + + buffer_t *buf = buffer_create_dynamic(pool_datastack_create(), 256); + + /* convert key to private key value */ + if (EVP_PKEY_base_id(pkey) == EVP_PKEY_RSA) { + unsigned char *ptr; + RSA *rsa = EVP_PKEY_get0_RSA(pkey); + int ln = i2d_RSAPrivateKey(rsa, &ptr); + if (ln < 1) + return dcrypt_openssl_error(error_r); + buffer_append(buf, ptr, ln); + } else if (EVP_PKEY_base_id(pkey) == EVP_PKEY_EC) { + unsigned char *ptr; + EC_KEY *eckey = EVP_PKEY_get0_EC_KEY(pkey); + const BIGNUM *pk = EC_KEY_get0_private_key(eckey); + /* serialize to MPI which is portable */ + int len = BN_bn2mpi(pk, NULL); + ptr = buffer_append_space_unsafe(buf, len); + BN_bn2mpi(pk, ptr); + } else { + /* Loading the key should have failed */ + i_unreached(); + } + + /* see if we want ECDH based or password based encryption */ + if (cipher != NULL && strncasecmp(cipher, "ecdh-", 5) == 0) { + i_assert(enc_key != NULL); + i_assert(password == NULL); + enctype = DCRYPT_DOVECOT_KEY_ENCRYPT_PK; + cipher2 = cipher+5; + } else if (cipher != NULL) { + i_assert(enc_key == NULL); + i_assert(password != NULL); + enctype = DCRYPT_DOVECOT_KEY_ENCRYPT_PASSWORD; + cipher2 = cipher; + } + + /* put in OID and encryption type */ + str_append(destination, t_strdup_printf("2\t%s\t%d\t", + objtxt, enctype)); + + /* perform encryption if desired */ + if (enctype > 0) { + if (!dcrypt_openssl_encrypt_private_key_dovecot(buf, enctype, cipher2, password, enc_key, destination, error_r)) { + buffer_set_used_size(destination, dest_used); + return FALSE; + } + } else { + binary_to_hex_append(destination, buf->data, buf->used); + } + + /* append public key id */ + str_append_c(destination, '\t'); + buffer_set_used_size(buf, 0); + bool res = dcrypt_openssl_private_key_id(key, "sha256", buf, error_r); + binary_to_hex_append(destination, buf->data, buf->used); + + if (!res) { + /* well, that didn't end well */ + buffer_set_used_size(destination, dest_used); + return FALSE; + } + return TRUE; +} + +static +bool dcrypt_openssl_store_public_key_dovecot(struct dcrypt_public_key *key, buffer_t *destination, const char **error_r) +{ + EVP_PKEY *pubkey = (EVP_PKEY*)key; + unsigned char *tmp = NULL; + size_t dest_used = buffer_get_used_size(destination); + + int rv = i2d_PUBKEY(pubkey, &tmp); + + if (tmp == NULL) + return dcrypt_openssl_error(error_r); + + /* then store it */ + str_append_c(destination, '2'); + str_append_c(destination, '\t'); + binary_to_hex_append(destination, tmp, rv); + OPENSSL_free(tmp); + + /* append public key ID */ + str_append_c(destination, '\t'); + + buffer_t *buf = buffer_create_dynamic(pool_datastack_create(), 32); + bool res = dcrypt_openssl_public_key_id(key, "sha256", buf, error_r); + + if (!res) { + buffer_set_used_size(destination, dest_used); + return FALSE; + } + + str_append(destination, binary_to_hex(buf->data, buf->used)); + return TRUE; +} + +static +bool dcrypt_openssl_load_private_key(struct dcrypt_private_key **key_r, enum dcrypt_key_format format, + const char *data, const char *password, struct dcrypt_private_key *dec_key, + const char **error_r) +{ + EVP_PKEY *key = NULL, *key2; + if (format == DCRYPT_FORMAT_DOVECOT) + return dcrypt_openssl_load_private_key_dovecot(key_r, data, password, dec_key, error_r); + + BIO *key_in = BIO_new_mem_buf((void*)data, strlen(data)); + + key = EVP_PKEY_new(); + + key2 = PEM_read_bio_PrivateKey(key_in, &key, NULL, (void*)password); + + BIO_vfree(key_in); + + if (key2 == NULL) { + EVP_PKEY_free(key); + return dcrypt_openssl_error(error_r); + } + + if (EVP_PKEY_base_id(key) == EVP_PKEY_EC) { + EC_KEY_set_conv_form(EVP_PKEY_get0_EC_KEY(key), POINT_CONVERSION_COMPRESSED); + } + + *key_r = (struct dcrypt_private_key *)key; + + return TRUE; +} + +static +bool dcrypt_openssl_load_public_key(struct dcrypt_public_key **key_r, enum dcrypt_key_format format, + const char *data, const char **error_r) +{ + EVP_PKEY *key = NULL; + if (format == DCRYPT_FORMAT_DOVECOT) + return dcrypt_openssl_load_public_key_dovecot(key_r, data, error_r); + + BIO *key_in = BIO_new_mem_buf((void*)data, strlen(data)); + if (key_in == NULL) + return dcrypt_openssl_error(error_r); + + key = PEM_read_bio_PUBKEY(key_in, &key, NULL, NULL); + if (BIO_reset(key_in) <= 0) i_unreached(); + if (key == NULL) { /* ec keys are bother */ + /* read the header */ + char buf[27]; /* begin public key */ + if (BIO_gets(key_in, buf, sizeof(buf)) != 1) { + BIO_vfree(key_in); + return dcrypt_openssl_error(error_r); + } + if (strcmp(buf, "-----BEGIN PUBLIC KEY-----") != 0) { + if (error_r != NULL) + *error_r = "Missing public key header"; + return FALSE; + } + BIO *b64 = BIO_new(BIO_f_base64()); + if (b64 == NULL) { + BIO_vfree(key_in); + return dcrypt_openssl_error(error_r); + } + EC_KEY *eckey = d2i_EC_PUBKEY_bio(b64, NULL); + if (eckey != NULL) { + EC_KEY_set_conv_form(eckey, POINT_CONVERSION_COMPRESSED); + EC_KEY_set_asn1_flag(eckey, OPENSSL_EC_NAMED_CURVE); + key = EVP_PKEY_new(); + if (key != NULL) + EVP_PKEY_set1_EC_KEY(key, eckey); + EC_KEY_free(eckey); + } + } + + BIO_vfree(key_in); + + if (key == NULL) + return dcrypt_openssl_error(error_r); + + *key_r = (struct dcrypt_public_key *)key; + + return TRUE; +} + +static +bool dcrypt_openssl_store_private_key(struct dcrypt_private_key *key, enum dcrypt_key_format format, + const char *cipher, buffer_t *destination, const char *password, struct dcrypt_public_key *enc_key, + const char **error_r) +{ + int ec; + if (format == DCRYPT_FORMAT_DOVECOT) { + bool ret; + ret = dcrypt_openssl_store_private_key_dovecot(key, cipher, destination, password, enc_key, error_r); + return ret; + } + + EVP_PKEY *pkey = (EVP_PKEY*)key; + BIO *key_out = BIO_new(BIO_s_mem()); + if (key_out == NULL) + return dcrypt_openssl_error(error_r); + + const EVP_CIPHER *algo = NULL; + if (cipher != NULL) { + algo = EVP_get_cipherbyname(cipher); + if (algo == NULL) { + if (error_r != NULL) + *error_r = t_strdup_printf("Invalid cipher %s", cipher); + return FALSE; + } + } + + ec = PEM_write_bio_PrivateKey(key_out, pkey, algo, NULL, 0, NULL, (void*)password); + + if (BIO_flush(key_out) <= 0) + ec = -1; + + if (ec != 1) { + BIO_vfree(key_out); + return dcrypt_openssl_error(error_r); + } + + long bs; + char *buf; + bs = BIO_get_mem_data(key_out, &buf); + buffer_append(destination, buf, bs); + BIO_vfree(key_out); + + return TRUE; +} + +static +bool dcrypt_openssl_store_public_key(struct dcrypt_public_key *key, enum dcrypt_key_format format, buffer_t *destination, const char **error_r) +{ + int ec; + if (format == DCRYPT_FORMAT_DOVECOT) + return dcrypt_openssl_store_public_key_dovecot(key, destination, error_r); + + EVP_PKEY *pkey = (EVP_PKEY*)key; + BIO *key_out = BIO_new(BIO_s_mem()); + if (key_out == NULL) + return dcrypt_openssl_error(error_r); + + BIO *b64; + if (EVP_PKEY_base_id(pkey) == EVP_PKEY_RSA) + ec = PEM_write_bio_PUBKEY(key_out, pkey); + else if ((b64 = BIO_new(BIO_f_base64())) == NULL) + ec = -1; + else { + (void)BIO_puts(key_out, "-----BEGIN PUBLIC KEY-----\n"); + (void)BIO_push(b64, key_out); + ec = i2d_EC_PUBKEY_bio(b64, EVP_PKEY_get0_EC_KEY(pkey)); + if (BIO_flush(b64) <= 0) + ec = -1; + (void)BIO_pop(b64); + BIO_vfree(b64); + if (BIO_puts(key_out, "-----END PUBLIC KEY-----") <= 0) + ec = -1; + } + + if (ec != 1) { + BIO_vfree(key_out); + return dcrypt_openssl_error(error_r); + } + + long bs; + char *buf; + bs = BIO_get_mem_data(key_out, &buf); + buffer_append(destination, buf, bs); + BIO_vfree(key_out); + + return TRUE; +} + +static +void dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r) +{ + EVP_PKEY *pkey = (EVP_PKEY*)priv_key; + EVP_PKEY *pk; + + if (*pub_key_r == NULL) { + pk = EVP_PKEY_new(); + i_assert(pk != NULL); /* we shouldn't get malloc() failures */ + } else + pk = (EVP_PKEY*)*pub_key_r; + + if (EVP_PKEY_base_id(pkey) == EVP_PKEY_RSA) + { + RSA *rsa = RSAPublicKey_dup(EVP_PKEY_get0_RSA(pkey)); + EVP_PKEY_set1_RSA(pk, rsa); + RSA_free(rsa); + } else if (EVP_PKEY_base_id(pkey) == EVP_PKEY_EC) { + EC_KEY* eck = EVP_PKEY_get1_EC_KEY(pkey); + EC_KEY_set_asn1_flag(eck, OPENSSL_EC_NAMED_CURVE); + EVP_PKEY_set1_EC_KEY(pk, eck); + EC_KEY_free(eck); + } else { + /* Loading the key should have failed */ + i_unreached(); + } + + *pub_key_r = (struct dcrypt_public_key*)pk; +} + +static +bool dcrypt_openssl_key_string_get_info(const char *key_data, enum dcrypt_key_format *format_r, enum dcrypt_key_version *version_r, + enum dcrypt_key_kind *kind_r, enum dcrypt_key_encryption_type *encryption_type_r, const char **encryption_key_hash_r, + const char **key_hash_r, const char **error_r) +{ + enum dcrypt_key_format format = DCRYPT_FORMAT_PEM; + enum dcrypt_key_version version = DCRYPT_KEY_VERSION_NA; + enum dcrypt_key_encryption_type encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_NONE; + enum dcrypt_key_kind kind = DCRYPT_KEY_KIND_PUBLIC; + char *encryption_key_hash = NULL; + char *key_hash = NULL; + + i_assert(key_data != NULL); + + /* is it PEM key */ + if (strncmp(key_data, "-----BEGIN ", 11) == 0) { + format = DCRYPT_FORMAT_PEM; + version = DCRYPT_KEY_VERSION_NA; + key_data += 11; + if (strncmp(key_data, "ENCRYPTED ", 10) == 0) { + encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_PASSWORD; + key_data += 10; + } + if (strncmp(key_data, "PRIVATE KEY-----", 16) == 0) + kind = DCRYPT_KEY_KIND_PRIVATE; + else if (strncmp(key_data, "PUBLIC KEY-----", 15) == 0) + kind = DCRYPT_KEY_KIND_PUBLIC; + else { + if (error_r != NULL) + *error_r = "Unknown/invalid PEM key type"; + return FALSE; + } + } else { + const char **fields = t_strsplit_tab(key_data); + int nfields = str_array_length(fields); + + if (nfields < 2) { + if (error_r != NULL) + *error_r = "Unknown key format"; + return FALSE; + } + + format = DCRYPT_FORMAT_DOVECOT; + + /* field 1 - version */ + if (strcmp(fields[0], "1") == 0) { + version = DCRYPT_KEY_VERSION_1; + if (nfields == 4) { + kind = DCRYPT_KEY_KIND_PUBLIC; + } else if (nfields == 5 && strcmp(fields[2],"0") == 0) { + kind = DCRYPT_KEY_KIND_PRIVATE; + encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_NONE; + } else if (nfields == 6 && strcmp(fields[2],"2") == 0) { + kind = DCRYPT_KEY_KIND_PRIVATE; + encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_PASSWORD; + } else if (nfields == 7 && strcmp(fields[2],"1") == 0) { + kind = DCRYPT_KEY_KIND_PRIVATE; + encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_KEY; + if (encryption_key_hash_r != NULL) + encryption_key_hash = i_strdup(fields[nfields-2]); + } else { + if (error_r != NULL) + *error_r = "Invalid dovecot v1 encoding"; + return FALSE; + } + } else if (strcmp(fields[0], "2") == 0) { + version = DCRYPT_KEY_VERSION_2; + if (nfields == 3) { + kind = DCRYPT_KEY_KIND_PUBLIC; + } else if (nfields == 5 && strcmp(fields[2],"0") == 0) { + kind = DCRYPT_KEY_KIND_PRIVATE; + encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_NONE; + } else if (nfields == 9 && strcmp(fields[2],"2") == 0) { + kind = DCRYPT_KEY_KIND_PRIVATE; + encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_PASSWORD; + } else if (nfields == 11 && strcmp(fields[2],"1") == 0) { + kind = DCRYPT_KEY_KIND_PRIVATE; + encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_KEY; + if (encryption_key_hash_r != NULL) + encryption_key_hash = i_strdup(fields[nfields-2]); + } else { + if (error_r != NULL) + *error_r = "Invalid dovecot v2 encoding"; + return FALSE; + } + } else { + if (error_r != NULL) + *error_r = "Invalid dovecot key version"; + return FALSE; + } + + /* last field is always key hash */ + if (key_hash_r != NULL) + key_hash = i_strdup(fields[nfields-1]); + } + + if (format_r != NULL) *format_r = format; + if (version_r != NULL) *version_r = version; + if (encryption_type_r != NULL) *encryption_type_r = encryption_type; + if (encryption_key_hash_r != NULL) { + *encryption_key_hash_r = t_strdup(encryption_key_hash); + i_free(encryption_key_hash); + } + if (kind_r != NULL) *kind_r = kind; + if (key_hash_r != NULL) { + *key_hash_r = t_strdup(key_hash); + i_free(key_hash); + } + return TRUE; +} + +static +void dcrypt_openssl_free_public_key(struct dcrypt_public_key **key) +{ + EVP_PKEY_free((EVP_PKEY*)*key); + *key = NULL; +} +static +void dcrypt_openssl_free_private_key(struct dcrypt_private_key **key) +{ + EVP_PKEY_free((EVP_PKEY*)*key); + *key = NULL; +} +static +void dcrypt_openssl_free_keypair(struct dcrypt_keypair *keypair) +{ + dcrypt_openssl_free_public_key(&(keypair->pub)); + dcrypt_openssl_free_private_key(&(keypair->priv)); +} + +static +bool dcrypt_openssl_rsa_encrypt(struct dcrypt_public_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r) +{ + int ec; + + EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new((EVP_PKEY*)key, NULL); + size_t outl = EVP_PKEY_size((EVP_PKEY*)key); + unsigned char buf[outl]; + + if (ctx == NULL || + EVP_PKEY_encrypt_init(ctx) < 1 || + EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) < 1 || + EVP_PKEY_encrypt(ctx, buf, &outl, data, data_len) < 1) { + dcrypt_openssl_error(error_r); + ec = -1; + } else { + buffer_append(result, buf, outl); + ec = 0; + } + + EVP_PKEY_CTX_free(ctx); + + return ec == 0; +} +static +bool dcrypt_openssl_rsa_decrypt(struct dcrypt_private_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r) +{ + int ec; + + EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new((EVP_PKEY*)key, NULL); + size_t outl = EVP_PKEY_size((EVP_PKEY*)key); + unsigned char buf[outl]; + + if (ctx == NULL || + EVP_PKEY_decrypt_init(ctx) < 1 || + EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) < 1 || + EVP_PKEY_decrypt(ctx, buf, &outl, data, data_len) < 1) { + dcrypt_openssl_error(error_r); + ec = -1; + } else { + buffer_append(result, buf, outl); + ec = 0; + } + + EVP_PKEY_CTX_free(ctx); + + return ec == 0; +} + +static +const char *dcrypt_openssl_oid2name(const unsigned char *oid, size_t oid_len, const char **error_r) +{ + const char *name; + ASN1_OBJECT *obj = d2i_ASN1_OBJECT(NULL, &oid, oid_len); + if (obj == NULL) { + dcrypt_openssl_error(error_r); + return NULL; + } + name = OBJ_nid2sn(OBJ_obj2nid(obj)); + ASN1_OBJECT_free(obj); + return name; +} + +static +bool dcrypt_openssl_name2oid(const char *name, buffer_t *oid, const char **error_r) +{ + ASN1_OBJECT *obj = OBJ_txt2obj(name, 0); + if (obj == NULL) + return dcrypt_openssl_error(error_r); + if (obj->length == 0) { + if (error_r != NULL) + *error_r = "Object has no OID assigned"; + return FALSE; + } + unsigned char *bufptr = buffer_append_space_unsafe(oid, obj->length + 2); + i2d_ASN1_OBJECT(obj, &bufptr); + ASN1_OBJECT_free(obj); + if (bufptr != NULL) { + return TRUE; + } + return dcrypt_openssl_error(error_r); +} + +static +enum dcrypt_key_type dcrypt_openssl_private_key_type(struct dcrypt_private_key *key) +{ + EVP_PKEY *priv = (EVP_PKEY*)key; + i_assert(priv != NULL); + if (EVP_PKEY_base_id(priv) == EVP_PKEY_RSA) return DCRYPT_KEY_RSA; + else if (EVP_PKEY_base_id(priv) == EVP_PKEY_EC) return DCRYPT_KEY_EC; + else i_unreached(); +} + +static +enum dcrypt_key_type dcrypt_openssl_public_key_type(struct dcrypt_public_key *key) +{ + EVP_PKEY *pub = (EVP_PKEY*)key; + i_assert(pub != NULL); + if (EVP_PKEY_base_id(pub) == EVP_PKEY_RSA) return DCRYPT_KEY_RSA; + else if (EVP_PKEY_base_id(pub) == EVP_PKEY_EC) return DCRYPT_KEY_EC; + else i_unreached(); +} + +/** this is the v1 old legacy way of doing key id's **/ +static +bool dcrypt_openssl_public_key_id_old(struct dcrypt_public_key *key, buffer_t *result, const char **error_r) +{ + unsigned char buf[SHA256_DIGEST_LENGTH]; + EVP_PKEY *pub = (EVP_PKEY*)key; + + i_assert(pub != NULL); + if (EVP_PKEY_base_id(pub) != EVP_PKEY_EC) { + if (error_r != NULL) + *error_r = "Only EC key supported"; + return FALSE; + } + + char *pub_pt_hex = ec_key_get_pub_point_hex(EVP_PKEY_get0_EC_KEY(pub)); + if (pub_pt_hex == NULL) + return dcrypt_openssl_error(error_r); + /* digest this */ + SHA256((const unsigned char*)pub_pt_hex, strlen(pub_pt_hex), buf); + buffer_append(result, buf, SHA256_DIGEST_LENGTH); + OPENSSL_free(pub_pt_hex); + return TRUE; +} + +static +bool dcrypt_openssl_private_key_id_old(struct dcrypt_private_key *key, buffer_t *result, const char **error_r) +{ + unsigned char buf[SHA256_DIGEST_LENGTH]; + EVP_PKEY *priv = (EVP_PKEY*)key; + + i_assert(priv != NULL); + if (EVP_PKEY_base_id(priv) != EVP_PKEY_EC) { + if (error_r != NULL) + *error_r = "Only EC key supported"; + return FALSE; + } + + char *pub_pt_hex = ec_key_get_pub_point_hex(EVP_PKEY_get0_EC_KEY(priv)); + if (pub_pt_hex == NULL) + return dcrypt_openssl_error(error_r); + /* digest this */ + SHA256((const unsigned char*)pub_pt_hex, strlen(pub_pt_hex), buf); + buffer_append(result, buf, SHA256_DIGEST_LENGTH); + OPENSSL_free(pub_pt_hex); + return TRUE; +} + +/** this is the new which uses H(der formatted public key) **/ +static +bool dcrypt_openssl_public_key_id_evp(EVP_PKEY *key, const EVP_MD *md, buffer_t *result, const char **error_r) +{ + bool res = FALSE; + unsigned char buf[EVP_MD_size(md)], *ptr; + + if (EVP_PKEY_base_id(key) == EVP_PKEY_EC) { + EC_KEY_set_conv_form(EVP_PKEY_get0_EC_KEY(key), POINT_CONVERSION_COMPRESSED); + } + BIO *b = BIO_new(BIO_s_mem()); + if (b == NULL || i2d_PUBKEY_bio(b, key) < 1) { + BIO_vfree(b); + return dcrypt_openssl_error(error_r); + } + long len = BIO_get_mem_data(b, &ptr); + unsigned int hlen = sizeof(buf); + /* then hash it */ +#if SSLEAY_VERSION_NUMBER >= 0x1010000fL + EVP_MD_CTX *ctx = EVP_MD_CTX_new(); +#else + EVP_MD_CTX *ctx = EVP_MD_CTX_create(); +#endif + if (ctx == NULL || + EVP_DigestInit_ex(ctx, md, NULL) < 1 || + EVP_DigestUpdate(ctx, (const unsigned char*)ptr, len) < 1 || + EVP_DigestFinal_ex(ctx, buf, &hlen) < 1) { + res = dcrypt_openssl_error(error_r); + } else { + buffer_append(result, buf, hlen); + res = TRUE; + } +#if SSLEAY_VERSION_NUMBER >= 0x1010000fL + EVP_MD_CTX_free(ctx); +#else + EVP_MD_CTX_destroy(ctx); +#endif + BIO_vfree(b); + + return res; +} + +static +bool dcrypt_openssl_public_key_id(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r) +{ + const EVP_MD *md = EVP_get_digestbyname(algorithm); + EVP_PKEY *pub = (EVP_PKEY*)key; + + i_assert(pub != NULL); + if (md == NULL) { + if (error_r != NULL) + *error_r = t_strdup_printf("Unknown cipher %s", algorithm); + return FALSE; + } + + return dcrypt_openssl_public_key_id_evp(pub, md, result, error_r); +} + +static +bool dcrypt_openssl_private_key_id(struct dcrypt_private_key *key, const char *algorithm, buffer_t *result, const char **error_r) +{ + const EVP_MD *md = EVP_get_digestbyname(algorithm); + EVP_PKEY *priv = (EVP_PKEY*)key; + + i_assert(priv != NULL); + if (md == NULL) { + if (error_r != NULL) + *error_r = t_strdup_printf("Unknown cipher %s", algorithm); + return FALSE; + } + + return dcrypt_openssl_public_key_id_evp(priv, md, result, error_r); +} + + +static struct dcrypt_vfs dcrypt_openssl_vfs = { + .initialize = dcrypt_openssl_initialize, + .ctx_sym_create = dcrypt_openssl_ctx_sym_create, + .ctx_sym_destroy = dcrypt_openssl_ctx_sym_destroy, + .ctx_sym_set_key = dcrypt_openssl_ctx_sym_set_key, + .ctx_sym_set_iv = dcrypt_openssl_ctx_sym_set_iv, + .ctx_sym_set_key_iv_random = dcrypt_openssl_ctx_sym_set_key_iv_random, + .ctx_sym_set_padding = dcrypt_openssl_ctx_sym_set_padding, + .ctx_sym_get_key = dcrypt_openssl_ctx_sym_get_key, + .ctx_sym_get_iv = dcrypt_openssl_ctx_sym_get_iv, + .ctx_sym_set_aad = dcrypt_openssl_ctx_sym_set_aad, + .ctx_sym_get_aad = dcrypt_openssl_ctx_sym_get_aad, + .ctx_sym_set_tag = dcrypt_openssl_ctx_sym_set_tag, + .ctx_sym_get_tag = dcrypt_openssl_ctx_sym_get_tag, + .ctx_sym_get_key_length = dcrypt_openssl_ctx_sym_get_key_length, + .ctx_sym_get_iv_length = dcrypt_openssl_ctx_sym_get_iv_length, + .ctx_sym_get_block_size = dcrypt_openssl_ctx_sym_get_block_size, + .ctx_sym_init = dcrypt_openssl_ctx_sym_init, + .ctx_sym_update = dcrypt_openssl_ctx_sym_update, + .ctx_sym_final = dcrypt_openssl_ctx_sym_final, + .ctx_hmac_create = dcrypt_openssl_ctx_hmac_create, + .ctx_hmac_destroy = dcrypt_openssl_ctx_hmac_destroy, + .ctx_hmac_set_key = dcrypt_openssl_ctx_hmac_set_key, + .ctx_hmac_set_key_random = dcrypt_openssl_ctx_hmac_set_key_random, + .ctx_hmac_get_digest_length = dcrypt_openssl_ctx_hmac_get_digest_length, + .ctx_hmac_get_key = dcrypt_openssl_ctx_hmac_get_key, + .ctx_hmac_init = dcrypt_openssl_ctx_hmac_init, + .ctx_hmac_update = dcrypt_openssl_ctx_hmac_update, + .ctx_hmac_final = dcrypt_openssl_ctx_hmac_final, + .ecdh_derive_secret_local = dcrypt_openssl_ecdh_derive_secret_local, + .ecdh_derive_secret_peer = dcrypt_openssl_ecdh_derive_secret_peer, + .pbkdf2 = dcrypt_openssl_pbkdf2, + .generate_keypair = dcrypt_openssl_generate_keypair, + .load_private_key = dcrypt_openssl_load_private_key, + .load_public_key = dcrypt_openssl_load_public_key, + .store_private_key = dcrypt_openssl_store_private_key, + .store_public_key = dcrypt_openssl_store_public_key, + .private_to_public_key = dcrypt_openssl_private_to_public_key, + .key_string_get_info = dcrypt_openssl_key_string_get_info, + .free_keypair = dcrypt_openssl_free_keypair, + .free_public_key = dcrypt_openssl_free_public_key, + .free_private_key = dcrypt_openssl_free_private_key, + .rsa_encrypt = dcrypt_openssl_rsa_encrypt, + .rsa_decrypt = dcrypt_openssl_rsa_decrypt, + .oid2name = dcrypt_openssl_oid2name, + .name2oid = dcrypt_openssl_name2oid, + .private_key_type = dcrypt_openssl_private_key_type, + .public_key_type = dcrypt_openssl_public_key_type, + .public_key_id = dcrypt_openssl_public_key_id, + .public_key_id_old = dcrypt_openssl_public_key_id_old, + .private_key_id = dcrypt_openssl_private_key_id, + .private_key_id_old = dcrypt_openssl_private_key_id_old, +}; + +void dcrypt_openssl_init(struct module *module ATTR_UNUSED) +{ + dovecot_openssl_common_global_ref(); + dcrypt_set_vfs(&dcrypt_openssl_vfs); +} + +void dcrypt_openssl_deinit(void) +{ + dovecot_openssl_common_global_unref(); +} diff --git a/src/lib-dcrypt/dcrypt-private.h b/src/lib-dcrypt/dcrypt-private.h new file mode 100644 index 00000000000..5e4a60d14c2 --- /dev/null +++ b/src/lib-dcrypt/dcrypt-private.h @@ -0,0 +1,108 @@ +#ifndef DCRYPT_PRIVATE_H +#define DCRYPT_PRIVATE_H + +#define DCRYPT_DOVECOT_KEY_ENCRYPT_HASH "sha256" +#define DCRYPT_DOVECOT_KEY_ENCRYPT_ROUNDS 2048 + +#define DCRYPT_DOVECOT_KEY_ENCRYPT_NONE 0 +#define DCRYPT_DOVECOT_KEY_ENCRYPT_PK 1 +#define DCRYPT_DOVECOT_KEY_ENCRYPT_PASSWORD 2 + +struct dcrypt_vfs { + bool (*initialize)(const struct dcrypt_settings *set, const char **error_r); + + bool (*ctx_sym_create)(const char *algorithm, + enum dcrypt_sym_mode mode, + struct dcrypt_context_symmetric **ctx_r, const char **error_r); + void (*ctx_sym_destroy)(struct dcrypt_context_symmetric **ctx); + + void (*ctx_sym_set_key)(struct dcrypt_context_symmetric *ctx, const unsigned char *key, size_t key_len); + void (*ctx_sym_set_iv)(struct dcrypt_context_symmetric *ctx, const unsigned char *iv, size_t iv_len); + void (*ctx_sym_set_key_iv_random)(struct dcrypt_context_symmetric *ctx); + + void (*ctx_sym_set_padding)(struct dcrypt_context_symmetric *ctx, bool padding); + + bool (*ctx_sym_get_key)(struct dcrypt_context_symmetric *ctx, buffer_t *key); + bool (*ctx_sym_get_iv)(struct dcrypt_context_symmetric *ctx, buffer_t *iv); + + void (*ctx_sym_set_aad)(struct dcrypt_context_symmetric *ctx, const unsigned char *aad, size_t aad_len); + bool (*ctx_sym_get_aad)(struct dcrypt_context_symmetric *ctx, buffer_t *aad); + void (*ctx_sym_set_tag)(struct dcrypt_context_symmetric *ctx, const unsigned char *tag, size_t tag_len); + bool (*ctx_sym_get_tag)(struct dcrypt_context_symmetric *ctx, buffer_t *tag); + + unsigned int (*ctx_sym_get_key_length)(struct dcrypt_context_symmetric *ctx); + unsigned int (*ctx_sym_get_iv_length)(struct dcrypt_context_symmetric *ctx); + unsigned int (*ctx_sym_get_block_size)(struct dcrypt_context_symmetric *ctx); + + bool (*ctx_sym_init)(struct dcrypt_context_symmetric *ctx, const char **error_r); + bool (*ctx_sym_update)(struct dcrypt_context_symmetric *ctx, const unsigned char *data, size_t data_len, + buffer_t *result, const char **error_r); + bool (*ctx_sym_final)(struct dcrypt_context_symmetric *ctx, buffer_t *result, const char **error_r); + + bool (*ctx_hmac_create)(const char *algorithm, struct dcrypt_context_hmac **ctx_r, const char **error_r); + void (*ctx_hmac_destroy)(struct dcrypt_context_hmac **ctx); + + void (*ctx_hmac_set_key)(struct dcrypt_context_hmac *ctx, const unsigned char *key, size_t key_len); + bool (*ctx_hmac_get_key)(struct dcrypt_context_hmac *ctx, buffer_t *key); + unsigned int (*ctx_hmac_get_digest_length)(struct dcrypt_context_hmac *ctx); + void (*ctx_hmac_set_key_random)(struct dcrypt_context_hmac *ctx); + + bool (*ctx_hmac_init)(struct dcrypt_context_hmac *ctx, const char **error_r); + bool (*ctx_hmac_update)(struct dcrypt_context_hmac *ctx, const unsigned char *data, size_t data_len, const char **error_r); + bool (*ctx_hmac_final)(struct dcrypt_context_hmac *ctx, buffer_t *result, const char **error_r); + + bool (*ecdh_derive_secret_local)(struct dcrypt_private_key *local_key, + buffer_t *R, buffer_t *S, const char **error_r); + bool (*ecdh_derive_secret_peer)(struct dcrypt_public_key *peer_key, + buffer_t *R, buffer_t *S, const char **error_r); + bool (*pbkdf2)(const unsigned char *password, size_t password_len, + const unsigned char *salt, size_t salt_len, const char *hash, + unsigned int rounds, buffer_t *result, unsigned int result_len, + const char **error_r); + + bool (*generate_keypair)(struct dcrypt_keypair *pair_r, enum dcrypt_key_type kind, + unsigned int bits, const char *curve, const char **error_r); + + bool (*load_private_key)(struct dcrypt_private_key **key_r, enum dcrypt_key_format format, const char *data, + const char *password, struct dcrypt_private_key *dec_key, const char **error_r); + bool (*load_public_key)(struct dcrypt_public_key **key_r, enum dcrypt_key_format format, const char *data, + const char **error_r); + + bool (*store_private_key)(struct dcrypt_private_key *key, enum dcrypt_key_format format, const char *cipher, buffer_t *destination, + const char *password, struct dcrypt_public_key *enc_key, const char **error_r); + bool (*store_public_key)(struct dcrypt_public_key *key, enum dcrypt_key_format format, buffer_t *destination, const char **error_r); + + void (*private_to_public_key)(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r); + + bool (*key_string_get_info)(const char *key_data, enum dcrypt_key_format *format_r, enum dcrypt_key_version *version_r, + enum dcrypt_key_kind *kind_r, enum dcrypt_key_encryption_type *encryption_type_r, const char **encryption_key_hash_r, + const char **key_hash_r, const char **error_r); + + void (*free_keypair)(struct dcrypt_keypair *keypair); + void (*free_public_key)(struct dcrypt_public_key **key); + void (*free_private_key)(struct dcrypt_private_key **key); + + bool (*rsa_encrypt)(struct dcrypt_public_key *key, const unsigned char *data, size_t data_len, + buffer_t *result, const char **error_r); + bool (*rsa_decrypt)(struct dcrypt_private_key *key, const unsigned char *data, size_t data_len, + buffer_t *result, const char **error_r); + + const char *(*oid2name)(const unsigned char *oid, size_t oid_len, const char **error_r); + bool (*name2oid)(const char *name, buffer_t *oid, const char **error_r); + + enum dcrypt_key_type (*private_key_type)(struct dcrypt_private_key *key); + enum dcrypt_key_type (*public_key_type)(struct dcrypt_public_key *key); + bool (*public_key_id)(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r); + bool (*public_key_id_old)(struct dcrypt_public_key *key, buffer_t *result, const char **error_r); + bool (*private_key_id)(struct dcrypt_private_key *key, const char *algorithm, buffer_t *result, const char **error_r); + bool (*private_key_id_old)(struct dcrypt_private_key *key, buffer_t *result, const char **error_r); +}; + +void dcrypt_set_vfs(struct dcrypt_vfs *vfs); + +void dcrypt_openssl_init(struct module *module ATTR_UNUSED); +void dcrypt_gnutls_init(struct module *module ATTR_UNUSED); +void dcrypt_openssl_deinit(void); +void dcrypt_gnutls_deinit(void); + +#endif diff --git a/src/lib-dcrypt/dcrypt.c b/src/lib-dcrypt/dcrypt.c new file mode 100644 index 00000000000..2269923c61b --- /dev/null +++ b/src/lib-dcrypt/dcrypt.c @@ -0,0 +1,288 @@ +#include "lib.h" +#include "module-dir.h" +#include "dcrypt.h" +#include "dcrypt-private.h" + +static struct module *dcrypt_module = NULL; +static struct dcrypt_vfs *dcrypt_vfs = NULL; +static const struct dcrypt_settings dcrypt_default_set; + +bool dcrypt_initialize(const char *backend, const struct dcrypt_settings *set, const char **error_r) +{ + struct module_dir_load_settings mod_set; + const char *error; + + if (dcrypt_vfs != NULL) { + return TRUE; + } + if (backend == NULL) backend = "openssl"; /* default for now */ + if (set == NULL) + set = &dcrypt_default_set; + + const char *implementation = t_strconcat("dcrypt_",backend,NULL); + + memset(&mod_set, 0, sizeof(mod_set)); + mod_set.abi_version = DOVECOT_ABI_VERSION; + mod_set.require_init_funcs = TRUE; + if (module_dir_try_load_missing(&dcrypt_module, DCRYPT_MODULE_DIR, + implementation, &mod_set, &error) < 0) { + if (error_r != NULL) + *error_r = error; + return FALSE; + } + module_dir_init(dcrypt_module); + i_assert(dcrypt_vfs != NULL); + if (dcrypt_vfs->initialize != NULL) { + if (!dcrypt_vfs->initialize(set, error_r)) { + dcrypt_deinitialize(); + return FALSE; + } + } + /* Destroy SSL module after(most of) the others. Especially lib-fs + backends may still want to access SSL module in their own + atexit-callbacks. */ + lib_atexit_priority(dcrypt_deinitialize, LIB_ATEXIT_PRIORITY_LOW); + return TRUE; +} + +void dcrypt_deinitialize(void) +{ + module_dir_unload(&dcrypt_module); + dcrypt_vfs = NULL; +} + +void dcrypt_set_vfs(struct dcrypt_vfs *vfs) +{ + dcrypt_vfs = vfs; +} + +bool dcrypt_ctx_sym_create(const char *algorithm, enum dcrypt_sym_mode mode, struct dcrypt_context_symmetric **ctx_r, const char **error_r) +{ + return dcrypt_vfs->ctx_sym_create(algorithm, mode, ctx_r, error_r); +} +void dcrypt_ctx_sym_destroy(struct dcrypt_context_symmetric **ctx) +{ + dcrypt_vfs->ctx_sym_destroy(ctx); +} + +void dcrypt_ctx_sym_set_key(struct dcrypt_context_symmetric *ctx, const unsigned char *key, size_t key_len) +{ + dcrypt_vfs->ctx_sym_set_key(ctx, key, key_len); +} +void dcrypt_ctx_sym_set_iv(struct dcrypt_context_symmetric *ctx, const unsigned char *iv, size_t iv_len) +{ + dcrypt_vfs->ctx_sym_set_iv(ctx, iv, iv_len); +} +void dcrypt_ctx_sym_set_key_iv_random(struct dcrypt_context_symmetric *ctx) +{ + dcrypt_vfs->ctx_sym_set_key_iv_random(ctx); +} + +bool dcrypt_ctx_sym_get_key(struct dcrypt_context_symmetric *ctx, buffer_t *key) +{ + return dcrypt_vfs->ctx_sym_get_key(ctx, key); +} +bool dcrypt_ctx_sym_get_iv(struct dcrypt_context_symmetric *ctx, buffer_t *iv) +{ + return dcrypt_vfs->ctx_sym_get_iv(ctx, iv); +} + +unsigned int dcrypt_ctx_sym_get_key_length(struct dcrypt_context_symmetric *ctx) +{ + return dcrypt_vfs->ctx_sym_get_key_length(ctx); +} + +unsigned int dcrypt_ctx_sym_get_iv_length(struct dcrypt_context_symmetric *ctx) +{ + return dcrypt_vfs->ctx_sym_get_iv_length(ctx); +} + +void dcrypt_ctx_sym_set_aad(struct dcrypt_context_symmetric *ctx, const unsigned char *aad, size_t aad_len) +{ + dcrypt_vfs->ctx_sym_set_aad(ctx, aad, aad_len); +} + +bool dcrypt_ctx_sym_get_aad(struct dcrypt_context_symmetric *ctx, buffer_t *aad) +{ + return dcrypt_vfs->ctx_sym_get_aad(ctx, aad); +} + +void dcrypt_ctx_sym_set_tag(struct dcrypt_context_symmetric *ctx, const unsigned char *tag, size_t tag_len) +{ + dcrypt_vfs->ctx_sym_set_tag(ctx, tag, tag_len); +} + +bool dcrypt_ctx_sym_get_tag(struct dcrypt_context_symmetric *ctx, buffer_t *tag) +{ + return dcrypt_vfs->ctx_sym_get_tag(ctx, tag); +} + +unsigned int dcrypt_ctx_sym_get_block_size(struct dcrypt_context_symmetric *ctx) { + return dcrypt_vfs->ctx_sym_get_block_size(ctx); +} + +bool dcrypt_ctx_sym_init(struct dcrypt_context_symmetric *ctx, const char **error_r) +{ + return dcrypt_vfs->ctx_sym_init(ctx, error_r); +} +bool dcrypt_ctx_sym_update(struct dcrypt_context_symmetric *ctx, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r) +{ + return dcrypt_vfs->ctx_sym_update(ctx, data, data_len, result, error_r); +} +bool dcrypt_ctx_sym_final(struct dcrypt_context_symmetric *ctx, buffer_t *result, const char **error_r) +{ + return dcrypt_vfs->ctx_sym_final(ctx, result, error_r); +} + +void dcrypt_ctx_sym_set_padding(struct dcrypt_context_symmetric *ctx, bool padding) +{ + dcrypt_vfs->ctx_sym_set_padding(ctx, padding); +} + +bool dcrypt_ctx_hmac_create(const char *algorithm, struct dcrypt_context_hmac **ctx_r, const char **error_r) +{ + return dcrypt_vfs->ctx_hmac_create(algorithm, ctx_r, error_r); +} +void dcrypt_ctx_hmac_destroy(struct dcrypt_context_hmac **ctx) +{ + dcrypt_vfs->ctx_hmac_destroy(ctx); +} + +void dcrypt_ctx_hmac_set_key(struct dcrypt_context_hmac *ctx, const unsigned char *key, size_t key_len) +{ + dcrypt_vfs->ctx_hmac_set_key(ctx, key, key_len); +} +bool dcrypt_ctx_hmac_get_key(struct dcrypt_context_hmac *ctx, buffer_t *key) +{ + return dcrypt_vfs->ctx_hmac_get_key(ctx, key); +} +void dcrypt_ctx_hmac_set_key_random(struct dcrypt_context_hmac *ctx) +{ + dcrypt_vfs->ctx_hmac_set_key_random(ctx); +} + +unsigned int dcrypt_ctx_hmac_get_digest_length(struct dcrypt_context_hmac *ctx) +{ + return dcrypt_vfs->ctx_hmac_get_digest_length(ctx); +} + +bool dcrypt_ctx_hmac_init(struct dcrypt_context_hmac *ctx, const char **error_r) +{ + return dcrypt_vfs->ctx_hmac_init(ctx, error_r); +} +bool dcrypt_ctx_hmac_update(struct dcrypt_context_hmac *ctx, const unsigned char *data, size_t data_len, const char **error_r) +{ + return dcrypt_vfs->ctx_hmac_update(ctx, data, data_len, error_r); +} +bool dcrypt_ctx_hmac_final(struct dcrypt_context_hmac *ctx, buffer_t *result, const char **error_r) +{ + return dcrypt_vfs->ctx_hmac_final(ctx, result, error_r); +} + +bool dcrypt_ecdh_derive_secret_local(struct dcrypt_private_key *local_key, buffer_t *R, buffer_t *S, const char **error_r) +{ + return dcrypt_vfs->ecdh_derive_secret_local(local_key, R, S, error_r); +} +bool dcrypt_ecdh_derive_secret_peer(struct dcrypt_public_key *peer_key, buffer_t *R, buffer_t *S, const char **error_r) +{ + return dcrypt_vfs->ecdh_derive_secret_peer(peer_key, R, S, error_r); +} + +bool dcrypt_pbkdf2(const unsigned char *password, size_t password_len, const unsigned char *salt, size_t salt_len, + const char *hash, unsigned int rounds, buffer_t *result, unsigned int result_len, const char **error_r) +{ + return dcrypt_vfs->pbkdf2(password, password_len, salt, salt_len, hash, rounds, result, result_len, error_r); +} + +bool dcrypt_keypair_generate(struct dcrypt_keypair *pair_r, enum dcrypt_key_type kind, unsigned int bits, const char *curve, const char **error_r) +{ + memset(pair_r, 0, sizeof(*pair_r)); + return dcrypt_vfs->generate_keypair(pair_r, kind, bits, curve, error_r); +} + +bool dcrypt_key_load_private(struct dcrypt_private_key **key_r, enum dcrypt_key_format format, const char *data, + const char *password, struct dcrypt_private_key *dec_key, const char **error_r) +{ + return dcrypt_vfs->load_private_key(key_r, format, data, password, dec_key, error_r); +} +bool dcrypt_key_load_public(struct dcrypt_public_key **key_r, enum dcrypt_key_format format, const char *data, const char **error_r) +{ + return dcrypt_vfs->load_public_key(key_r, format, data, error_r); +} +bool dcrypt_key_store_private(struct dcrypt_private_key *key, enum dcrypt_key_format format, const char *cipher, buffer_t *destination, + const char *password, struct dcrypt_public_key *enc_key, const char **error_r) +{ + return dcrypt_vfs->store_private_key(key, format, cipher, destination, password, enc_key, error_r); +} +bool dcrypt_key_store_public(struct dcrypt_public_key *key, enum dcrypt_key_format format, buffer_t *destination, const char **error_r) +{ + return dcrypt_vfs->store_public_key(key, format, destination, error_r); +} + +void dcrypt_key_convert_private_to_public(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r) +{ + dcrypt_vfs->private_to_public_key(priv_key, pub_key_r); +} +bool dcrypt_key_string_get_info(const char *key_data, enum dcrypt_key_format *format_r, enum dcrypt_key_version *version_r, + enum dcrypt_key_kind *kind_r, enum dcrypt_key_encryption_type *encryption_type_r, const char **encryption_key_hash_r, + const char **key_hash_r, const char **error_r) +{ + return dcrypt_vfs->key_string_get_info(key_data, format_r, version_r, kind_r, encryption_type_r, + encryption_key_hash_r, key_hash_r, error_r); +} + +enum dcrypt_key_type dcrypt_key_type_private(struct dcrypt_private_key *key) +{ + return dcrypt_vfs->private_key_type(key); +} +enum dcrypt_key_type dcrypt_key_type_public(struct dcrypt_public_key *key) +{ + return dcrypt_vfs->public_key_type(key); +} +bool dcrypt_key_id_public(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r) +{ + return dcrypt_vfs->public_key_id(key, algorithm, result, error_r); +} +bool dcrypt_key_id_public_old(struct dcrypt_public_key *key, buffer_t *result, const char **error_r) +{ + return dcrypt_vfs->public_key_id_old(key, result, error_r); +} +bool dcrypt_key_id_private(struct dcrypt_private_key *key, const char *algorithm, buffer_t *result, const char **error_r) +{ + return dcrypt_vfs->private_key_id(key, algorithm, result, error_r); +} +bool dcrypt_key_id_private_old(struct dcrypt_private_key *key, buffer_t *result, const char **error_r) +{ + return dcrypt_vfs->private_key_id_old(key, result, error_r); +} +void dcrypt_keypair_free(struct dcrypt_keypair *keypair) +{ + dcrypt_vfs->free_keypair(keypair); +} +void dcrypt_key_free_public(struct dcrypt_public_key **key) +{ + dcrypt_vfs->free_public_key(key); +} +void dcrypt_key_free_private(struct dcrypt_private_key **key) +{ + dcrypt_vfs->free_private_key(key); +} + +bool dcrypt_rsa_encrypt(struct dcrypt_public_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r) +{ + return dcrypt_vfs->rsa_encrypt(key, data, data_len, result, error_r); +} +bool dcrypt_rsa_decrypt(struct dcrypt_private_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r) +{ + return dcrypt_vfs->rsa_decrypt(key, data, data_len, result, error_r); +} + +const char *dcrypt_oid2name(const unsigned char *oid, size_t oid_len, const char **error_r) +{ + return dcrypt_vfs->oid2name(oid, oid_len, error_r); +} +bool dcrypt_name2oid(const char *name, buffer_t *oid, const char **error_r) +{ + return dcrypt_vfs->name2oid(name, oid, error_r); +} + diff --git a/src/lib-dcrypt/dcrypt.h b/src/lib-dcrypt/dcrypt.h new file mode 100644 index 00000000000..f6d3e44ed64 --- /dev/null +++ b/src/lib-dcrypt/dcrypt.h @@ -0,0 +1,211 @@ +#ifndef DCRYPT_H +#define DCRYPT_H 1 + +struct dcrypt_context_symmetric; +struct dcrypt_context_hmac; +struct dcrypt_public_key; +struct dcrypt_private_key; + +struct dcrypt_keypair { + struct dcrypt_public_key *pub; + struct dcrypt_private_key *priv; +}; + +enum dcrypt_sym_mode { + DCRYPT_MODE_ENCRYPT, + DCRYPT_MODE_DECRYPT +}; + +enum dcrypt_key_type { + DCRYPT_KEY_RSA = 0x1, + DCRYPT_KEY_EC = 0x2 +}; + +/** + * dovecot key format: + * version tab version-specific data + * v1: version tab nid tab raw ec private key (in hex) + * v2: version tab algorithm oid tab private-or-public-key-only (in hex) + */ +enum dcrypt_key_format { + DCRYPT_FORMAT_PEM, + DCRYPT_FORMAT_DOVECOT, +}; + +enum dcrypt_key_encryption_type { + DCRYPT_KEY_ENCRYPTION_TYPE_NONE, + DCRYPT_KEY_ENCRYPTION_TYPE_KEY, + DCRYPT_KEY_ENCRYPTION_TYPE_PASSWORD +}; + +enum dcrypt_key_version { + DCRYPT_KEY_VERSION_1, + DCRYPT_KEY_VERSION_2, + DCRYPT_KEY_VERSION_NA /* not applicable, PEM key */ +}; + +enum dcrypt_key_kind { + DCRYPT_KEY_KIND_PUBLIC, + DCRYPT_KEY_KIND_PRIVATE +}; + +struct dcrypt_settings { + /* OpenSSL engine to use */ + const char *crypto_device; +}; + +/** + * load and initialize dcrypt backend, use either openssl or gnutls + */ +bool dcrypt_initialize(const char *backend, const struct dcrypt_settings *set, const char **error_r); + +/** + * deinitialize dcrypt + */ +void dcrypt_deinitialize(void); + +/** + * create symmetric context + */ +bool dcrypt_ctx_sym_create(const char *algorithm, enum dcrypt_sym_mode mode, struct dcrypt_context_symmetric **ctx_r, const char **error_r); + +/** + * destroy symmetric context and free memory + */ +void dcrypt_ctx_sym_destroy(struct dcrypt_context_symmetric **ctx); + +/** + * key and IV manipulation functions + */ +void dcrypt_ctx_sym_set_key(struct dcrypt_context_symmetric *ctx, const unsigned char *key, size_t key_len); +void dcrypt_ctx_sym_set_iv(struct dcrypt_context_symmetric *ctx, const unsigned char *iv, size_t iv_len); +void dcrypt_ctx_sym_set_key_iv_random(struct dcrypt_context_symmetric *ctx); +bool dcrypt_ctx_sym_get_key(struct dcrypt_context_symmetric *ctx, buffer_t *key); +bool dcrypt_ctx_sym_get_iv(struct dcrypt_context_symmetric *ctx, buffer_t *iv); + +/** + * turn padding on/off (default: on) + */ +void dcrypt_ctx_sym_set_padding(struct dcrypt_context_symmetric *ctx, bool padding); + + +/** + * authentication data manipulation (use with GCM only) + */ +void dcrypt_ctx_sym_set_aad(struct dcrypt_context_symmetric *ctx, const unsigned char *aad, size_t aad_len); +bool dcrypt_ctx_sym_get_aad(struct dcrypt_context_symmetric *ctx, buffer_t *aad); +/** + * result tag from aead (use with GCM only) + */ +void dcrypt_ctx_sym_set_tag(struct dcrypt_context_symmetric *ctx, const unsigned char *tag, size_t tag_len); +bool dcrypt_ctx_sym_get_tag(struct dcrypt_context_symmetric *ctx, buffer_t *tag); + +/* get various lengths */ +unsigned int dcrypt_ctx_sym_get_key_length(struct dcrypt_context_symmetric *ctx); +unsigned int dcrypt_ctx_sym_get_iv_length(struct dcrypt_context_symmetric *ctx); +unsigned int dcrypt_ctx_sym_get_block_size(struct dcrypt_context_symmetric *ctx); + +/** + * initialize crypto + */ +bool dcrypt_ctx_sym_init(struct dcrypt_context_symmetric *ctx, const char **error_r); +/** + * update with data + */ +bool dcrypt_ctx_sym_update(struct dcrypt_context_symmetric *ctx, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r); +/** + * perform final step (may or may not emit data) + */ +bool dcrypt_ctx_sym_final(struct dcrypt_context_symmetric *ctx, buffer_t *result, const char **error_r); + +/** + * create HMAC context, algorithm is digest algorithm + */ +bool dcrypt_ctx_hmac_create(const char *algorithm, struct dcrypt_context_hmac **ctx_r, const char **error_r); +/** + * destroy HMAC context and free memory + */ +void dcrypt_ctx_hmac_destroy(struct dcrypt_context_hmac **ctx); + +/** + * hmac key manipulation + */ +void dcrypt_ctx_hmac_set_key(struct dcrypt_context_hmac *ctx, const unsigned char *key, size_t key_len); +bool dcrypt_ctx_hmac_get_key(struct dcrypt_context_hmac *ctx, buffer_t *key); +void dcrypt_ctx_hmac_set_key_random(struct dcrypt_context_hmac *ctx); + +/** + * get digest length for HMAC + */ +unsigned int dcrypt_ctx_hmac_get_digest_length(struct dcrypt_context_hmac *ctx); + +/** + * initialize hmac + */ +bool dcrypt_ctx_hmac_init(struct dcrypt_context_hmac *ctx, const char **error_r); +/** + * update hmac context with data + */ +bool dcrypt_ctx_hmac_update(struct dcrypt_context_hmac *ctx, const unsigned char *data, size_t data_len, const char **error_r); +/** + * perform final rounds and retrieve result + */ +bool dcrypt_ctx_hmac_final(struct dcrypt_context_hmac *ctx, buffer_t *result, const char **error_r); + + +/** + * Elliptic Curve based Diffie-Heffman shared secret derivation */ +bool dcrypt_ecdh_derive_secret_local(struct dcrypt_private_key *local_key, buffer_t *R, buffer_t *S, const char **error_r); +bool dcrypt_ecdh_derive_secret_peer(struct dcrypt_public_key *peer_key, buffer_t *R, buffer_t *S, const char **error_r); + +/** + * generate cryptographic data from password and salt. Use 1000-10000 for rounds. + */ +bool dcrypt_pbkdf2(const unsigned char *password, size_t password_len, const unsigned char *salt, size_t salt_len, + const char *hash, unsigned int rounds, buffer_t *result, unsigned int result_len, const char **error_r); + +bool dcrypt_keypair_generate(struct dcrypt_keypair *pair_r, enum dcrypt_key_type kind, unsigned int bits, const char *curve, const char **error_r); + +/** + * load loads key structure from external format. + * store stores key structure into external format. + * + * you can provide either PASSWORD or ENC_KEY, not both. + */ +bool dcrypt_key_load_private(struct dcrypt_private_key **key_r, enum dcrypt_key_format format, const char *data, + const char *password, struct dcrypt_private_key *dec_key, const char **error_r); + +bool dcrypt_key_load_public(struct dcrypt_public_key **key_r, enum dcrypt_key_format format, const char *data, const char **error_r); + +bool dcrypt_key_store_private(struct dcrypt_private_key *key, enum dcrypt_key_format format, const char *cipher, + buffer_t *destination, const char *password, struct dcrypt_public_key *enc_key, const char **error_r); + +bool dcrypt_key_store_public(struct dcrypt_public_key *key, enum dcrypt_key_format format, buffer_t *destination, const char **error_r); + +void dcrypt_key_convert_private_to_public(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r); + +void dcrypt_keypair_free(struct dcrypt_keypair *keypair); + +void dcrypt_key_free_public(struct dcrypt_public_key **key); +void dcrypt_key_free_private(struct dcrypt_private_key **key); + +enum dcrypt_key_type dcrypt_key_type_private(struct dcrypt_private_key *key); +enum dcrypt_key_type dcrypt_key_type_public(struct dcrypt_public_key *key); +bool dcrypt_key_id_public(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r); /* return digest of key */ +bool dcrypt_key_id_public_old(struct dcrypt_public_key *key, buffer_t *result, const char **error_r); /* return SHA1 sum of key */ +bool dcrypt_key_id_private(struct dcrypt_private_key *key, const char *algorithm, buffer_t *result, const char **error_r); /* return digest of key */ +bool dcrypt_key_id_private_old(struct dcrypt_private_key *key, buffer_t *result, const char **error_r); /* return SHA1 sum of key */ + +bool dcrypt_key_string_get_info(const char *key_data, enum dcrypt_key_format *format_r, enum dcrypt_key_version *version_r, + enum dcrypt_key_kind *kind_r, enum dcrypt_key_encryption_type *encryption_type_r, const char **encryption_key_hash_r, + const char **key_hash_r, const char **error_r); + +/* RSA stuff */ +bool dcrypt_rsa_encrypt(struct dcrypt_public_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r); +bool dcrypt_rsa_decrypt(struct dcrypt_private_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r); + +/* OID stuff */ +const char *dcrypt_oid2name(const unsigned char *oid, size_t oid_len, const char **error_r); +bool dcrypt_name2oid(const char *name, buffer_t *oid, const char **error_r); + +#endif diff --git a/src/lib-dcrypt/istream-decrypt.c b/src/lib-dcrypt/istream-decrypt.c new file mode 100644 index 00000000000..ae39d0d5ab0 --- /dev/null +++ b/src/lib-dcrypt/istream-decrypt.c @@ -0,0 +1,869 @@ +#include "lib.h" +#include "buffer.h" +#include "randgen.h" +#include "safe-memset.h" +#include "hash-method.h" +#include "sha2.h" +#include "dcrypt.h" +#include "istream.h" +#include "istream-decrypt.h" +#include "istream-private.h" +#include "dcrypt-iostream-private.h" + +#include "hex-binary.h" + +#include + +#define ISTREAM_DECRYPT_READ_FIRST 15 + +enum io_stream_encrypt_flags { + IO_STREAM_ENC_INTEGRITY_HMAC = 0x1, + IO_STREAM_ENC_INTEGRITY_AEAD = 0x2, + IO_STREAM_ENC_INTEGRITY_NONE = 0x4, + IO_STREAM_ENC_VERSION_1 = 0x8, +}; + +struct decrypt_istream { + struct istream_private istream; + buffer_t *buf; + + i_stream_decrypt_get_key_callback_t *key_callback; + void *key_context; + + struct dcrypt_private_key *priv_key; + bool initialized; + bool finalized; + bool use_mac; + + uoff_t ftr, pos; + enum io_stream_encrypt_flags flags; + + unsigned char *iv; /* original iv, in case seeking is done, future feature */ + + struct dcrypt_context_symmetric *ctx_sym; + struct dcrypt_context_hmac *ctx_mac; + + enum { + DECRYPT_FORMAT_V1, + DECRYPT_FORMAT_V2 + } format; +}; + +static +ssize_t i_stream_decrypt_read_header_v1(struct decrypt_istream *stream, + const unsigned char *data, size_t mlen) +{ + const char *error = NULL; + size_t hdr_len = 0; + uint16_t len; + int ec, i = 0; + + const unsigned char *digest_pos = NULL, *key_digest_pos = NULL, *key_ct_pos = NULL; + + size_t pos = 9; + size_t digest_len = 0; + size_t key_ct_len = 0; + size_t key_digest_size = 0; + + buffer_t ephemeral_key; + buffer_t *secret = buffer_create_dynamic(pool_datastack_create(), 256); + buffer_t *key = buffer_create_dynamic(pool_datastack_create(), 256); + + hdr_len = ((data[0] << 8) | data[1]) + 12; + + if (mlen < hdr_len - pos) { + /* try to read more */ + return 0; + } + + data+=2; + mlen-=2; + + memcpy(&len, data, 2); + + while(i < 4 && mlen > 2 && (len = ntohs(len)) <= (mlen - 2) && len > 0) { + data += 2; + mlen -= 2; + pos += 2; + + switch(i++) { + case 0: + buffer_create_from_const_data(&ephemeral_key, data, len); + break; + case 1: + /* public key id */ + digest_pos = data; + digest_len = len; + break; + case 2: + /* encryption key digest */ + key_digest_pos = data; + key_digest_size = len; + break; + case 3: + /* encrypted key data */ + key_ct_pos = data; + key_ct_len = len; + break; + } + pos += len; + data += len; + mlen -= len; + memcpy(&len, data, 2); + } + + if (i < 4) { + io_stream_set_error(&stream->istream.iostream, "Invalid or corrupted header"); + return -1; + } + + /* we don't have a private key */ + if (stream->priv_key == NULL) { + /* see if we can get one */ + if (stream->key_callback != NULL) { + const char *key_id = binary_to_hex(digest_pos, digest_len); + int ret = stream->key_callback(key_id, &(stream->priv_key), &error, stream->key_context); + if (ret < 0) { + io_stream_set_error(&stream->istream.iostream, "Private key not available: %s", error); + return -1; + } + if (ret == 0) { + io_stream_set_error(&stream->istream.iostream, "Private key not available"); + return -1; + } + } else { + io_stream_set_error(&stream->istream.iostream, "Private key not available"); + return -1; + } + } + + buffer_t *check = buffer_create_dynamic(pool_datastack_create(), 32); + + if (!dcrypt_key_id_private_old(stream->priv_key, check, &error)) { + io_stream_set_error(&stream->istream.iostream, "Cannot get public key hash: %s", error); + return -1; + } else { + if (memcmp(digest_pos, check->data, I_MIN(digest_len,check->used)) != 0) { + io_stream_set_error(&stream->istream.iostream, "Private key not available"); + return -1; + } + } + + /* derive shared secret */ + if (!dcrypt_ecdh_derive_secret_local(stream->priv_key, &ephemeral_key, secret, &error)) { + io_stream_set_error(&stream->istream.iostream, "Cannot perform ECDH: %s", error); + return -1; + } + + /* run it thru SHA256 once */ + const struct hash_method *hash = &hash_method_sha256; + unsigned char hctx[hash->context_size]; + unsigned char hres[hash->digest_size]; + hash->init(hctx); + hash->loop(hctx, secret->data, secret->used); + hash->result(hctx, hres); + safe_memset(buffer_get_modifiable_data(secret, 0), 0, secret->used); + + /* NB! The old code was broken and used this kind of IV - it is not correct, but + we need to stay compatible with old data */ + + /* use it to decrypt the actual encryption key */ + struct dcrypt_context_symmetric *dctx; + if (!dcrypt_ctx_sym_create("aes-256-ctr", DCRYPT_MODE_DECRYPT, &dctx, &error)) { + io_stream_set_error(&stream->istream.iostream, "Key decryption error: %s", error); + return -1; + } + + ec = 0; + dcrypt_ctx_sym_set_iv(dctx, (const unsigned char*)"\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0", 16); + dcrypt_ctx_sym_set_key(dctx, hres, hash->digest_size); + if (!dcrypt_ctx_sym_init(dctx, &error) || + !dcrypt_ctx_sym_update(dctx, key_ct_pos, key_ct_len, key, &error) || + !dcrypt_ctx_sym_final(dctx, key, &error)) { + io_stream_set_error(&stream->istream.iostream, "Key decryption error: %s", error); + ec = -1; + } + dcrypt_ctx_sym_destroy(&dctx); + + if (ec != 0) { + io_stream_set_error(&stream->istream.iostream, "Key decryption error: %s", error); + return -1; + } + + /* see if we got the correct key */ + hash->init(hctx); + hash->loop(hctx, key->data, key->used); + hash->result(hctx, hres); + + if (key_digest_size != sizeof(hres)) { + io_stream_set_error(&stream->istream.iostream, "Key decryption error: invalid digest length"); + return -1; + } + if (memcmp(hres, key_digest_pos, sizeof(hres)) != 0) { + io_stream_set_error(&stream->istream.iostream, "Key decryption error: decrypted key is invalid"); + return -1; + } + + /* prime context with key */ + if (!dcrypt_ctx_sym_create("aes-256-ctr", DCRYPT_MODE_DECRYPT, &(stream->ctx_sym), &error)) { + io_stream_set_error(&stream->istream.iostream, "Decryption context create error: %s", error); + return -1; + } + + /* Again, old code used this IV, so we have to use it too */ + dcrypt_ctx_sym_set_iv(stream->ctx_sym, (const unsigned char*)"\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0", 16); + dcrypt_ctx_sym_set_key(stream->ctx_sym, key->data, key->used); + + safe_memset(buffer_get_modifiable_data(key, 0), 0, key->used); + + if (!dcrypt_ctx_sym_init(stream->ctx_sym, &error)) { + io_stream_set_error(&stream->istream.iostream, "Decryption init error: %s", error); + return -1; + } + + stream->use_mac = FALSE; + stream->initialized = TRUE; + /* now we are ready to decrypt stream */ + + return hdr_len; +} + +static bool get_msb32(const unsigned char **_data, const unsigned char *end, uint32_t *num_r) +{ + const unsigned char *data = *_data; + if (end-data < 4) + return FALSE; + *num_r = ((uint32_t)data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3]; + *_data += 4; + return TRUE; +} + +static +bool i_stream_decrypt_der(const unsigned char **_data, + const unsigned char *end, const char **str_r) +{ + const unsigned char *data = *_data; + unsigned int len; + + if (end-data < 2) + return FALSE; + /* get us DER encoded length */ + if ((data[1] & 0x80) != 0) { + /* two byte length */ + if (end-data < 3) + return FALSE; + len = ((data[1] & 0x7f) << 8) + data[2] + 3; + } else { + len = data[1] + 2; + } + if ((size_t)(end-data) < len) + return FALSE; + *str_r = dcrypt_oid2name(data, len, NULL); + *_data += len; + return TRUE; +} + +static +ssize_t i_stream_decrypt_key(struct decrypt_istream *stream, const char *malg, unsigned int rounds, + const unsigned char *data, const unsigned char *end, buffer_t *key, size_t key_len) +{ + const char *error; + enum dcrypt_key_type ktype; + int keys; + bool have_key = FALSE; + unsigned char dgst[32]; + uint32_t val; + buffer_t buf; + + if (data == end) + return 0; + + keys = *data++; + + /* if we have a key, prefab the digest */ + if (stream->key_callback == NULL) { + if (stream->priv_key == NULL) { + io_stream_set_error(&stream->istream.iostream, "Decryption error: no private key available"); + return -1; + } + buffer_create_from_data(&buf, dgst, sizeof(dgst)); + dcrypt_key_id_private(stream->priv_key, "sha256", &buf, NULL); + } + + /* for each key */ + for(;keys>0;keys--) { + if ((size_t)(end-data) < 1 + (ssize_t)sizeof(dgst)) + return 0; + ktype = *data++; + + if (stream->key_callback != NULL) { + const char *hexdgst = binary_to_hex(data, sizeof(dgst)); /* digest length */ + /* hope you going to give us right key.. */ + int ret = stream->key_callback(hexdgst, &(stream->priv_key), &error, stream->key_context); + if (ret < 0) { + io_stream_set_error(&stream->istream.iostream, "Private key not available: %s", error); + return -1; + } + if (ret > 0) { + have_key = TRUE; + break; + } + } else { + /* see if key matches to the one we have */ + if (memcmp(dgst, data, sizeof(dgst)) == 0) { + have_key = TRUE; + break; + } + } + data += sizeof(dgst); + + /* wasn't correct key, skip over some data */ + if (!get_msb32(&data, end, &val) || + !get_msb32(&data, end, &val)) + return 0; + } + + /* didn't find matching key */ + if (!have_key) { + io_stream_set_error(&stream->istream.iostream, "Decryption error: no private key available"); + return -1; + } + + data += sizeof(dgst); + + const unsigned char *ephemeral_key; + uint32_t ep_key_len; + const unsigned char *encrypted_key; + uint32_t eklen; + const unsigned char *ekhash; + uint32_t ekhash_len; + + /* read ephemeral key (can be missing for RSA) */ + if (!get_msb32(&data, end, &ep_key_len) || (size_t)(end-data) < ep_key_len) + return 0; + ephemeral_key = data; + data += ep_key_len; + + /* read encrypted key */ + if (!get_msb32(&data, end, &eklen) || (size_t)(end-data) < eklen) + return 0; + encrypted_key = data; + data += eklen; + + /* read key data hash */ + if (!get_msb32(&data, end, &ekhash_len) || (size_t)(end-data) < ekhash_len) + return 0; + ekhash = data; + data += ekhash_len; + + /* decrypt the seed */ + if (ktype == DCRYPT_KEY_RSA) { + if (!dcrypt_rsa_decrypt(stream->priv_key, encrypted_key, eklen, key, &error)) { + io_stream_set_error(&stream->istream.iostream, "key decryption error: %s", error); + return -1; + } + } else if (ktype == DCRYPT_KEY_EC) { + /* perform ECDHE */ + buffer_t *temp_key = buffer_create_dynamic(pool_datastack_create(), 256); + buffer_t *secret = buffer_create_dynamic(pool_datastack_create(), 256); + buffer_t peer_key; + buffer_create_from_const_data(&peer_key, ephemeral_key, ep_key_len); + if (!dcrypt_ecdh_derive_secret_local(stream->priv_key, &peer_key, secret, &error)) { + io_stream_set_error(&stream->istream.iostream, "Key decryption error: corrupted header"); + return -1; + } + + /* use shared secret and peer key to generate decryption key, AES-256-CBC has 32 byte key and 16 byte IV */ + if (!dcrypt_pbkdf2(secret->data, secret->used, peer_key.data, peer_key.used, + malg, rounds, temp_key, 32+16, &error)) { + safe_memset(buffer_get_modifiable_data(secret, 0), 0, secret->used); + io_stream_set_error(&stream->istream.iostream, "Key decryption error: %s", error); + return -1; + } + + safe_memset(buffer_get_modifiable_data(secret, 0), 0, secret->used); + if (temp_key->used != 32+16) { + safe_memset(buffer_get_modifiable_data(temp_key, 0), 0, temp_key->used); + io_stream_set_error(&stream->istream.iostream, "Cannot perform key decryption: invalid temporary key"); + return -1; + } + struct dcrypt_context_symmetric *dctx; + if (!dcrypt_ctx_sym_create("AES-256-CBC", DCRYPT_MODE_DECRYPT, &dctx, &error)) { + safe_memset(buffer_get_modifiable_data(temp_key, 0), 0, temp_key->used); + io_stream_set_error(&stream->istream.iostream, "Key decryption error: %s", error); + return -1; + } + const unsigned char *ptr = temp_key->data; + + /* we use ephemeral_key for IV */ + dcrypt_ctx_sym_set_key(dctx, ptr, 32); + dcrypt_ctx_sym_set_iv(dctx, ptr+32, 16); + safe_memset(buffer_get_modifiable_data(temp_key, 0), 0, temp_key->used); + + int ec = 0; + if (!dcrypt_ctx_sym_init(dctx, &error) || + !dcrypt_ctx_sym_update(dctx, encrypted_key, eklen, key, &error) || + !dcrypt_ctx_sym_final(dctx, key, &error)) { + io_stream_set_error(&stream->istream.iostream, "Cannot perform key decryption: %s", error); + ec = -1; + } + + if (key->used != key_len) { + io_stream_set_error(&stream->istream.iostream, "Cannot perform key decryption: invalid key length"); + ec = -1; + } + + dcrypt_ctx_sym_destroy(&dctx); + if (ec != 0) return ec; + } else { + io_stream_set_error(&stream->istream.iostream, "Decryption error: unsupported key type 0x%02x", ktype); + return -1; + } + + /* make sure we were able to decrypt the encrypted key correctly */ + const struct hash_method *hash = hash_method_lookup(t_str_lcase(malg)); + if (hash == NULL) { + safe_memset(buffer_get_modifiable_data(key, 0), 0, key->used); + io_stream_set_error(&stream->istream.iostream, "Decryption error: unsupported hash algorithm: %s", malg); + return -1; + } + unsigned char hctx[hash->context_size]; + unsigned char hres[hash->digest_size]; + hash->init(hctx); + hash->loop(hctx, key->data, key->used); + hash->result(hctx, hres); + + for(int i = 1; i < 2049; i++) { + uint32_t i_msb = htonl(i); + + hash->init(hctx); + hash->loop(hctx, hres, sizeof(hres)); + hash->loop(hctx, &i_msb, sizeof(i_msb)); + hash->result(hctx, hres); + } + + /* do the comparison */ + if (memcmp(ekhash, hres, I_MIN(ekhash_len, sizeof(hres))) != 0) { + safe_memset(buffer_get_modifiable_data(key, 0), 0, key->used); + io_stream_set_error(&stream->istream.iostream, "Decryption error: corrupted header ekhash"); + return -1; + } + return 1; +} + +static +int i_stream_decrypt_header_contents(struct decrypt_istream *stream, + const unsigned char *data, size_t size) +{ + const unsigned char *end = data + size; + bool failed = FALSE; + + /* read cipher OID */ + const char *calg; + if (!i_stream_decrypt_der(&data, end, &calg)) + return 0; + if (calg == NULL || !dcrypt_ctx_sym_create(calg, DCRYPT_MODE_DECRYPT, &(stream->ctx_sym), NULL)) { + io_stream_set_error(&stream->istream.iostream, "Decryption error: unsupported/invalid cipher: %s", calg); + return -1; + } + + /* read MAC oid (MAC is used for PBKDF2 and key data digest, too) */ + const char *malg; + if (!i_stream_decrypt_der(&data, end, &malg)) + return 0; + if (malg == NULL || !dcrypt_ctx_hmac_create(malg, &(stream->ctx_mac), NULL)) { + io_stream_set_error(&stream->istream.iostream, "Decryption error: unsupported/invalid MAC algorithm: %s", malg); + return -1; + } + + /* read rounds (for PBKDF2) */ + uint32_t rounds; + if (!get_msb32(&data, end, &rounds)) + return 0; + /* read key data length */ + uint32_t kdlen; + if (!get_msb32(&data, end, &kdlen)) + return 0; + + size_t tagsize; + + if ((stream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) { + tagsize = IOSTREAM_TAG_SIZE; + } else if ((stream->flags & IO_STREAM_ENC_INTEGRITY_AEAD) == IO_STREAM_ENC_INTEGRITY_AEAD) { + tagsize = IOSTREAM_TAG_SIZE; + } else { + tagsize = 0; + } + + /* how much key data we should be getting */ + size_t kl = dcrypt_ctx_sym_get_key_length(stream->ctx_sym) + dcrypt_ctx_sym_get_iv_length(stream->ctx_sym) + tagsize; + buffer_t *keydata = buffer_create_dynamic(pool_datastack_create(), kl); + + /* try to decrypt the keydata with a private key */ + int ret; + if ((ret = i_stream_decrypt_key(stream, malg, rounds, data, end, keydata, kl)) <= 0) + return ret; + + /* oh, it worked! */ + const unsigned char *ptr = keydata->data; + if (keydata->used != kl) { + /* but returned wrong amount of data */ + io_stream_set_error(&stream->istream.iostream, "Key decryption error: Key data length mismatch"); + return -1; + } + + /* prime contexts */ + dcrypt_ctx_sym_set_key(stream->ctx_sym, ptr, dcrypt_ctx_sym_get_key_length(stream->ctx_sym)); + ptr += dcrypt_ctx_sym_get_key_length(stream->ctx_sym); + dcrypt_ctx_sym_set_iv(stream->ctx_sym, ptr, dcrypt_ctx_sym_get_iv_length(stream->ctx_sym)); + stream->iv = i_malloc(dcrypt_ctx_sym_get_iv_length(stream->ctx_sym)); + memcpy(stream->iv, ptr, dcrypt_ctx_sym_get_iv_length(stream->ctx_sym)); + ptr += dcrypt_ctx_sym_get_iv_length(stream->ctx_sym); + + /* based on the chosen MAC, initialize HMAC or AEAD */ + if ((stream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) { + const char *error; + dcrypt_ctx_hmac_set_key(stream->ctx_mac, ptr, tagsize); + if (!dcrypt_ctx_hmac_init(stream->ctx_mac, &error)) { + io_stream_set_error(&stream->istream.iostream, "MAC error: %s", error); + failed = TRUE; + } + stream->ftr = dcrypt_ctx_hmac_get_digest_length(stream->ctx_mac); + stream->use_mac = TRUE; + } else if ((stream->flags & IO_STREAM_ENC_INTEGRITY_AEAD) == IO_STREAM_ENC_INTEGRITY_AEAD) { + dcrypt_ctx_sym_set_aad(stream->ctx_sym, ptr, tagsize); + stream->ftr = tagsize; + stream->use_mac = TRUE; + } else { + stream->use_mac = FALSE; + } + /* destroy private key data */ + safe_memset(buffer_get_modifiable_data(keydata, 0), 0, keydata->used); + buffer_set_used_size(keydata, 0); + return failed ? -1 : 1; +} + +static +ssize_t i_stream_decrypt_read_header(struct decrypt_istream *stream, + const unsigned char *data, size_t mlen) +{ + const char *error; + const unsigned char *end = data + mlen; + + /* check magic */ + if (mlen < sizeof(IOSTREAM_CRYPT_MAGIC)) + return 0; + if (memcmp(data, IOSTREAM_CRYPT_MAGIC, sizeof(IOSTREAM_CRYPT_MAGIC)) != 0) { + io_stream_set_error(&stream->istream.iostream, "Invalid magic"); + return -1; + } + data += sizeof(IOSTREAM_CRYPT_MAGIC); + + if (data >= end) + return 0; /* read more? */ + + /* check version */ + if (*data == '\x01') { + stream->format = DECRYPT_FORMAT_V1; + return i_stream_decrypt_read_header_v1(stream, data+1, end - (data+1)); + } else if (*data != '\x02') { + io_stream_set_error(&stream->istream.iostream, "Unsupported encrypted data 0x%02x", *data); + return -1; + } + + stream->format = DECRYPT_FORMAT_V2; + + data++; + + /* read flags */ + uint32_t flags; + if (!get_msb32(&data, end, &flags)) + return 0; + stream->flags = flags; + + /* get the total length of header */ + uint32_t hdr_len; + if (!get_msb32(&data, end, &hdr_len)) + return 0; + /* do not forget stream format */ + if ((size_t)(end-data)+1 < hdr_len) + return 0; + + int ret; + if ((ret = i_stream_decrypt_header_contents(stream, data, hdr_len)) < 0) + return -1; + else if (ret == 0) { + io_stream_set_error(&stream->istream.iostream, "Decryption error: truncate header length"); + return -1; + } + stream->initialized = TRUE; + + /* if it all went well, try to initialize decryption context */ + if (!dcrypt_ctx_sym_init(stream->ctx_sym, &error)) { + io_stream_set_error(&stream->istream.iostream, "Decryption init error: %s", error); + return -1; + } + return hdr_len; +} + +static ssize_t +i_stream_decrypt_read(struct istream_private *stream) +{ + struct decrypt_istream *dstream = + (struct decrypt_istream *)stream; + const unsigned char *data; + size_t size, decrypt_size; + const char *error = NULL; + int ret; + bool check_mac = FALSE; + + /* not if it's broken */ + if (stream->istream.stream_errno != 0) + return -1; + + for (;;) { + /* remove skipped data from buffer */ + if (stream->skip > 0) { + i_assert(stream->skip <= dstream->buf->used); + buffer_delete(dstream->buf, 0, stream->skip); + stream->pos -= stream->skip; + stream->skip = 0; + } + + stream->buffer = dstream->buf->data; + + i_assert(stream->pos <= dstream->buf->used); + if (stream->pos >= dstream->istream.max_buffer_size) { + /* stream buffer still at maximum */ + return -2; + } + + /* if something is already decrypted, return as much of it as + we can */ + if (dstream->buf->used > 0) { + size_t new_pos, bytes; + + /* only return up to max_buffer_size bytes, even when buffer + actually has more, as not to confuse the caller */ + if (dstream->buf->used <= dstream->istream.max_buffer_size) { + new_pos = dstream->buf->used; + if (dstream->finalized) + stream->istream.eof = TRUE; + } else { + new_pos = dstream->istream.max_buffer_size; + } + + bytes = new_pos - stream->pos; + stream->pos = new_pos; + return (ssize_t)bytes; + } + if (dstream->finalized) { + /* all data decrypted */ + stream->istream.eof = TRUE; + return -1; + } + /* need to read more input */ + ret = i_stream_read(stream->parent); + if (ret == 0) + return 0; + data = i_stream_get_data(stream->parent, &size); + + if (ret == -1 && (size == 0 || stream->parent->stream_errno != 0)) { + stream->istream.stream_errno = stream->parent->stream_errno; + + /* file was empty */ + if (!dstream->initialized && size == 0 && stream->parent->eof) { + stream->istream.eof = TRUE; + return -1; + } + + if (stream->istream.stream_errno != 0) + return -1; + + if (!dstream->initialized) { + io_stream_set_error(&stream->iostream, + "Decryption error: %s", + "Input truncated in decryption header"); + stream->istream.stream_errno = EINVAL; + return -1; + } + + /* final block */ + if (dcrypt_ctx_sym_final(dstream->ctx_sym, + dstream->buf, &error)) { + dstream->finalized = TRUE; + continue; + } + io_stream_set_error(&stream->iostream, + "MAC error: %s", error); + stream->istream.stream_errno = EINVAL; + return -1; + } + + if (!dstream->initialized) { + ssize_t hret; + if ((hret=i_stream_decrypt_read_header + (dstream, data, size)) <= 0) { + if (hret < 0) { + stream->istream.stream_errno = EINVAL; + } + return hret; + } + i_stream_skip(stream->parent, hret); + data = i_stream_get_data(stream->parent, &size); + } + decrypt_size = size; + + if (dstream->use_mac) { + if (stream->parent->eof) { + if (decrypt_size < dstream->ftr) { + io_stream_set_error(&stream->iostream, + "Decryption error: footer is longer than data"); + stream->istream.stream_errno = EINVAL; + return -1; + } + check_mac = TRUE; + } else { + /* ignore footer's length of data until we + reach EOF */ + size -= dstream->ftr; + } + decrypt_size -= dstream->ftr; + if ((dstream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) { + if (!dcrypt_ctx_hmac_update(dstream->ctx_mac, + data, decrypt_size, &error)) { + io_stream_set_error(&stream->iostream, + "MAC error: %s", error); + return -1; + } + } + } + + if (check_mac) { + if ((dstream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) { + unsigned char dgst[dcrypt_ctx_hmac_get_digest_length(dstream->ctx_mac)]; + buffer_t db; + buffer_create_from_data(&db, dgst, sizeof(dgst)); + if (!dcrypt_ctx_hmac_final(dstream->ctx_mac, &db, &error)) { + io_stream_set_error(&stream->iostream, + "Cannot verify MAC: %s", error); + } + if (memcmp(dgst, data + decrypt_size, dcrypt_ctx_hmac_get_digest_length(dstream->ctx_mac)) != 0) { + io_stream_set_error(&stream->iostream, + "Cannot verify MAC: mismatch"); + return -1; + } + } else if ((dstream->flags & IO_STREAM_ENC_INTEGRITY_AEAD) == IO_STREAM_ENC_INTEGRITY_AEAD) { + dcrypt_ctx_sym_set_tag(dstream->ctx_sym, data + decrypt_size, dstream->ftr); + } + } + + if (!dcrypt_ctx_sym_update(dstream->ctx_sym, + data, decrypt_size, dstream->buf, &error)) { + io_stream_set_error(&stream->iostream, + "Decryption error: %s", error); + stream->istream.stream_errno = EINVAL; + return -1; + } + i_stream_skip(stream->parent, size); + } +} + +static +void i_stream_decrypt_close(struct iostream_private *stream, + bool close_parent) +{ + struct decrypt_istream *dstream = + (struct decrypt_istream *)stream; + + if (close_parent) + i_stream_close(dstream->istream.parent); +} + +static +void i_stream_decrypt_destroy(struct iostream_private *stream) +{ + struct decrypt_istream *dstream = + (struct decrypt_istream *)stream; + + if (dstream->buf != NULL) + buffer_free(&dstream->buf); + if (dstream->iv != NULL) + i_free_and_null(dstream->iv); + if (dstream->ctx_sym != NULL) + dcrypt_ctx_sym_destroy(&(dstream->ctx_sym)); + if (dstream->ctx_mac != NULL) + dcrypt_ctx_hmac_destroy(&(dstream->ctx_mac)); + + i_stream_unref(&(dstream->istream.parent)); +} + +static +struct decrypt_istream *i_stream_create_decrypt_common(struct istream *input) +{ + struct decrypt_istream *dstream; + + dstream = i_new(struct decrypt_istream, 1); + dstream->istream.max_buffer_size = input->real_stream->max_buffer_size; + dstream->istream.read = i_stream_decrypt_read; + dstream->istream.iostream.close = i_stream_decrypt_close; + dstream->istream.iostream.destroy = i_stream_decrypt_destroy; + + dstream->istream.istream.readable_fd = FALSE; + dstream->istream.istream.blocking = TRUE; + dstream->istream.istream.seekable = FALSE; + + dstream->buf = buffer_create_dynamic(default_pool, 128); + + (void)i_stream_create(&dstream->istream, input, + i_stream_get_fd(input)); + return dstream; +} + +struct istream * +i_stream_create_decrypt(struct istream *input, struct dcrypt_private_key *priv_key) +{ + struct decrypt_istream *dstream; + + dstream = i_stream_create_decrypt_common(input); + dstream->priv_key = priv_key; + return &dstream->istream.istream; +} + +struct istream * +i_stream_create_sym_decrypt(struct istream *input, struct dcrypt_context_symmetric *ctx) +{ + const char *error; + int ec; + struct decrypt_istream *dstream; + dstream = i_stream_create_decrypt_common(input); + dstream->use_mac = FALSE; + dstream->initialized = TRUE; + + if (!dcrypt_ctx_sym_init(ctx, &error)) ec = -1; + else ec = 0; + + dstream->ctx_sym = ctx; + + if (ec != 0) { + io_stream_set_error(&dstream->istream.iostream, "Cannot initialize decryption: %s", error); + dstream->istream.istream.stream_errno = EINVAL; + }; + + return &dstream->istream.istream; +} + +struct istream * +i_stream_create_decrypt_callback(struct istream *input, + i_stream_decrypt_get_key_callback_t *callback, + void *context) +{ + struct decrypt_istream *dstream; + + i_assert(callback != NULL); + + dstream = i_stream_create_decrypt_common(input); + dstream->key_callback = callback; + dstream->key_context = context; + return &dstream->istream.istream; +} diff --git a/src/lib-dcrypt/istream-decrypt.h b/src/lib-dcrypt/istream-decrypt.h new file mode 100644 index 00000000000..0c59bc8ee82 --- /dev/null +++ b/src/lib-dcrypt/istream-decrypt.h @@ -0,0 +1,30 @@ +#ifndef ISTREAM_DECRYPT_H +#define ISTREAM_DECRYPT_H + +struct dcrypt_private_key; +struct dcrypt_context_symmetric; + +/* Look for a private key for a specified public key digest and set it to + priv_key_r. Returns 1 if ok, 0 if key doesn't exist, -1 on internal error. */ +typedef int +i_stream_decrypt_get_key_callback_t(const char *pubkey_digest, + struct dcrypt_private_key **priv_key_r, + const char **error_r, void *context); + +struct istream * +i_stream_create_decrypt(struct istream *input, struct dcrypt_private_key *priv_key); + +/* create stream for reading plain encrypted data with no header or MAC. + do not call dcrypt_ctx_sym_init + */ +struct istream * +i_stream_create_sym_decrypt(struct istream *input, struct dcrypt_context_symmetric *ctx); + + +/* Decrypt the istream. When a private key is needed, the callback will be + called. This allows using multiple private keys for different mails. */ +struct istream * +i_stream_create_decrypt_callback(struct istream *input, + i_stream_decrypt_get_key_callback_t *callback, + void *context); +#endif diff --git a/src/lib-dcrypt/ostream-encrypt.c b/src/lib-dcrypt/ostream-encrypt.c new file mode 100644 index 00000000000..d5c20a728df --- /dev/null +++ b/src/lib-dcrypt/ostream-encrypt.c @@ -0,0 +1,689 @@ +/* file truct dcrypt_public_keyyntax + * magic (14 bytes) + * version (1 bytes) + * flags (4 bytes) + * size of header (4 bytes) + * sha1 of key id (20 bytes) + * cipher oid + * mac oid + * rounds (4 bytes) + * key data size (4 bytes) + * key data + * cipher data + * mac data (mac specific bytes) + */ + +#include "lib.h" +#include "buffer.h" +#include "randgen.h" +#include "ostream-encrypt.h" +#include "ostream-private.h" +#include "hash-method.h" +#include "sha2.h" +#include "safe-memset.h" +#include "dcrypt.h" +#include "dcrypt-iostream-private.h" + +#include + +#define IO_STREAM_ENCRYPT_SEED_SIZE 32 +#define IO_STREAM_ENCRYPT_ROUNDS 2048 + +struct encrypt_ostream { + struct ostream_private ostream; + + struct dcrypt_context_symmetric *ctx_sym; + struct dcrypt_context_hmac *ctx_mac; + + enum io_stream_encrypt_flags flags; + struct dcrypt_public_key *pub; + + unsigned char *key_data; + size_t key_data_len; + + buffer_t *cipher_oid; + buffer_t *mac_oid; + size_t block_size; + + bool finalized; + bool failed; + bool prefix_written; +}; + +static +int o_stream_encrypt_send(struct encrypt_ostream *stream, + const unsigned char *data, size_t size) +{ + ssize_t ec; + + ec = o_stream_send(stream->ostream.parent, data, size); + if (ec == (ssize_t)size) + return 0; + else if (ec < 0) { + o_stream_copy_error_from_parent(&stream->ostream); + return -1; + } else { + io_stream_set_error(&stream->ostream.iostream, + "ostream-encrypt: Unexpectedly short write to parent stream"); + stream->ostream.ostream.stream_errno = EINVAL; + return -1; + } +} + +static +int o_stream_encrypt_send_header_v1(struct encrypt_ostream *stream) +{ + unsigned char c; + unsigned short s; + + i_assert(!stream->prefix_written); + stream->prefix_written = TRUE; + + buffer_t *values = buffer_create_dynamic(pool_datastack_create(), 256); + buffer_append(values, IOSTREAM_CRYPT_MAGIC, sizeof(IOSTREAM_CRYPT_MAGIC)); + /* version */ + c = 1; + buffer_append(values, &c, 1); + /* header length including this and data written so far */ + s = htons(stream->key_data_len); + buffer_append(values, &s, 2); + /* then write key data */ + buffer_append(values, stream->key_data, stream->key_data_len); + i_free_and_null(stream->key_data); + + /* then send it to stream */ + return o_stream_encrypt_send(stream, values->data, values->used); +} + +static +int o_stream_encrypt_send_header_v2(struct encrypt_ostream *stream) +{ + unsigned char c; + unsigned int i; + + i_assert(!stream->prefix_written); + stream->prefix_written = TRUE; + + buffer_t *values = buffer_create_dynamic(pool_datastack_create(), 256); + buffer_append(values, IOSTREAM_CRYPT_MAGIC, sizeof(IOSTREAM_CRYPT_MAGIC)); + c = 2; + buffer_append(values, &c, 1); + i = htonl(stream->flags); + buffer_append(values, &i, 4); + /* store total length of header + 9 = version + flags + length + 8 = rounds + key data length + */ + i = htonl(sizeof(IOSTREAM_CRYPT_MAGIC) + 9 + stream->cipher_oid->used + stream->mac_oid->used + 8 + stream->key_data_len); + buffer_append(values, &i, 4); + + buffer_append_buf(values, stream->cipher_oid, 0, (size_t)-1); + buffer_append_buf(values, stream->mac_oid, 0, (size_t)-1); + i = htonl(IO_STREAM_ENCRYPT_ROUNDS); + buffer_append(values, &i, 4); + i = htonl(stream->key_data_len); + buffer_append(values, &i, 4); + buffer_append(values, stream->key_data, stream->key_data_len); + i_free_and_null(stream->key_data); + + return o_stream_encrypt_send(stream, values->data, values->used); +} + +static +int o_stream_encrypt_keydata_create_v1(struct encrypt_ostream *stream) +{ + buffer_t *encrypted_key, *ephemeral_key, *secret, *res, buf; + const char *error = NULL; + const struct hash_method *hash = &hash_method_sha256; + + /* various temporary buffers */ + unsigned char seed[IO_STREAM_ENCRYPT_SEED_SIZE]; + unsigned char pkhash[hash->digest_size]; + unsigned char ekhash[hash->digest_size]; + unsigned char hres[hash->digest_size]; + + unsigned char hctx[hash->context_size]; + + /* hash the public key first */ + buffer_create_from_data(&buf, pkhash, sizeof(pkhash)); + if (!dcrypt_key_id_public_old(stream->pub, &buf, &error)) { + io_stream_set_error(&stream->ostream.iostream, "Key hash failed: %s", error); + return -1; + } + + /* hash the key base */ + hash->init(hctx); + hash->loop(hctx, seed, sizeof(seed)); + hash->result(hctx, ekhash); + + ephemeral_key = buffer_create_dynamic(pool_datastack_create(), 256); + encrypted_key = buffer_create_dynamic(pool_datastack_create(), 256); + secret = buffer_create_dynamic(pool_datastack_create(), 256); + + if (!dcrypt_ecdh_derive_secret_peer(stream->pub, ephemeral_key, secret, &error)) { + io_stream_set_error(&stream->ostream.iostream, "Cannot perform ECDH: %s", error); + return -1; + } + + /* hash the secret data */ + hash->init(hctx); + hash->loop(hctx, secret->data, secret->used); + hash->result(hctx, hres); + safe_memset(buffer_get_modifiable_data(secret, 0), 0, secret->used); + + /* use it to encrypt the actual encryption key */ + struct dcrypt_context_symmetric *dctx; + if (!dcrypt_ctx_sym_create("aes-256-ctr", DCRYPT_MODE_ENCRYPT, &dctx, &error)) { + io_stream_set_error(&stream->ostream.iostream, "Key encryption error: %s", error); + return -1; + } + + random_fill(seed, sizeof(seed)); + hash->init(hctx); + hash->loop(hctx, seed, sizeof(seed)); + hash->result(hctx, ekhash); + + int ec = 0; + + /* NB! The old code was broken and used this kind of IV - it is not correct, but + we need to stay compatible with old data */ + dcrypt_ctx_sym_set_iv(dctx, (const unsigned char*)"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 16); + dcrypt_ctx_sym_set_key(dctx, hres, sizeof(hres)); + + if (!dcrypt_ctx_sym_init(dctx, &error) || + !dcrypt_ctx_sym_update(dctx, seed, sizeof(seed), encrypted_key, &error) || + !dcrypt_ctx_sym_final(dctx, encrypted_key, &error)) { + ec = -1; + } + dcrypt_ctx_sym_destroy(&dctx); + + if (ec != 0) { + safe_memset(seed, 0, sizeof(seed)); + io_stream_set_error(&stream->ostream.iostream, "Key encryption error: %s", error); + return -1; + } + + if (ec == 0) { + /* same as above */ + dcrypt_ctx_sym_set_iv(stream->ctx_sym, (const unsigned char*)"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 16); + dcrypt_ctx_sym_set_key(stream->ctx_sym, seed, sizeof(seed)); + } + safe_memset(seed, 0, sizeof(seed)); + + if (ec != 0) { + io_stream_set_error(&stream->ostream.iostream, "Encryption init error: %s", error); + return -1; + } + + if (!dcrypt_ctx_sym_init(stream->ctx_sym, &error)) { + io_stream_set_error(&stream->ostream.iostream, "Encryption init error: %s", error); + return -1; + } + + res = buffer_create_dynamic(default_pool, 256); + + /* ephemeral key */ + unsigned short s; + s = htons(ephemeral_key->used); + buffer_append(res, &s, 2); + buffer_append(res, ephemeral_key->data, ephemeral_key->used); + /* public key hash */ + s = htons(sizeof(pkhash)); + buffer_append(res, &s, 2); + buffer_append(res, pkhash, sizeof(pkhash)); + /* encrypted key hash */ + s = htons(sizeof(ekhash)); + buffer_append(res, &s, 2); + buffer_append(res, ekhash, sizeof(ekhash)); + /* encrypted key */ + s = htons(encrypted_key->used); + buffer_append(res, &s, 2); + buffer_append(res, encrypted_key->data, encrypted_key->used); + + stream->key_data_len = res->used; + stream->key_data = buffer_free_without_data(&res); + + return 0; +} + +static +int o_stream_encrypt_key_for_pubkey_v2(struct encrypt_ostream *stream, const char *malg, + const unsigned char *key, size_t key_len, struct dcrypt_public_key *pubkey, buffer_t *res) +{ + enum dcrypt_key_type ktype; + const char *error; + buffer_t *encrypted_key, *ephemeral_key, *temp_key; + + ephemeral_key = buffer_create_dynamic(pool_datastack_create(), 256); + encrypted_key = buffer_create_dynamic(pool_datastack_create(), 256); + temp_key = buffer_create_dynamic(pool_datastack_create(), 48); + + ktype = dcrypt_key_type_public(pubkey); + + if (ktype == DCRYPT_KEY_RSA) { + /* encrypt key as R (as we don't need DH with RSA)*/ + if (!dcrypt_rsa_encrypt(pubkey, key, key_len, encrypted_key, &error)) { + io_stream_set_error(&stream->ostream.iostream, "Cannot encrypt key data: %s", error); + return -1; + } + } else if (ktype == DCRYPT_KEY_EC) { + /* R = our ephemeral public key */ + buffer_t *secret = buffer_create_dynamic(pool_datastack_create(), 256); + + /* derive ephemeral key and shared secret */ + if (!dcrypt_ecdh_derive_secret_peer(pubkey, ephemeral_key, secret, &error)) { + io_stream_set_error(&stream->ostream.iostream, "Cannot perform ECDH: %s", error); + return -1; + } + + /* use shared secret and ephemeral key to generate encryption key/iv */ + if (!dcrypt_pbkdf2(secret->data, secret->used, ephemeral_key->data, ephemeral_key->used, + malg, IO_STREAM_ENCRYPT_ROUNDS, temp_key, 48, &error)) { + safe_memset(buffer_get_modifiable_data(secret, 0), 0, secret->used); + io_stream_set_error(&stream->ostream.iostream, "Cannot perform key encryption: %s", error); + } + safe_memset(buffer_get_modifiable_data(secret, 0), 0, secret->used); + + /* encrypt key with shared secret */ + struct dcrypt_context_symmetric *dctx; + if (!dcrypt_ctx_sym_create("AES-256-CBC", DCRYPT_MODE_ENCRYPT, &dctx, &error)) { + safe_memset(buffer_get_modifiable_data(temp_key, 0), 0, temp_key->used); + io_stream_set_error(&stream->ostream.iostream, "Cannot perform key encryption: %s", error); + return -1; + } + + const unsigned char *ptr = temp_key->data; + i_assert(temp_key->used == 48); + + dcrypt_ctx_sym_set_key(dctx, ptr, 32); + dcrypt_ctx_sym_set_iv(dctx, ptr+32, 16); + safe_memset(buffer_get_modifiable_data(temp_key, 0), 0, temp_key->used); + + int ec = 0; + if (!dcrypt_ctx_sym_init(dctx, &error) || + !dcrypt_ctx_sym_update(dctx, key, key_len, encrypted_key, &error) || + !dcrypt_ctx_sym_final(dctx, encrypted_key, &error)) { + io_stream_set_error(&stream->ostream.iostream, "Cannot perform key encryption: %s", error); + ec = -1; + } + + dcrypt_ctx_sym_destroy(&dctx); + if (ec != 0) return ec; + } else { + io_stream_set_error(&stream->ostream.iostream, "Unsupported key type"); + return -1; + } + + /* store key type */ + char kt = ktype; + buffer_append(res, &kt, 1); + /* store hash of public key as ID */ + dcrypt_key_id_public(stream->pub, "sha256", res, NULL); + /* store ephemeral key (if present) */ + unsigned int val = htonl(ephemeral_key->used); + buffer_append(res, &val, 4); + buffer_append_buf(res, ephemeral_key, 0, (size_t)-1); + /* store encrypted key */ + val = htonl(encrypted_key->used); + buffer_append(res, &val, 4); + buffer_append_buf(res, encrypted_key, 0, (size_t)-1); + + return 0; +} + +static +int o_stream_encrypt_keydata_create_v2(struct encrypt_ostream *stream, const char *malg) +{ + const struct hash_method *hash = hash_method_lookup(malg); + const char *error; + size_t tagsize; + const unsigned char *ptr; + size_t kl; + unsigned int val; + + buffer_t *keydata, *res; + + if (hash == NULL) { + io_stream_set_error(&stream->ostream.iostream, + "Encryption init error: Hash algorithm '%s' not supported", malg); + return -1; + } + + /* key data length for internal use */ + if ((stream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) { + tagsize = IOSTREAM_TAG_SIZE; + } else if ((stream->flags & IO_STREAM_ENC_INTEGRITY_AEAD) == IO_STREAM_ENC_INTEGRITY_AEAD) { + tagsize = IOSTREAM_TAG_SIZE; + } else { + /* do not include MAC */ + tagsize = 0; + } + + /* generate keydata length of random data for key/iv/mac */ + kl = dcrypt_ctx_sym_get_key_length(stream->ctx_sym) + dcrypt_ctx_sym_get_iv_length(stream->ctx_sym) + tagsize; + keydata = buffer_create_dynamic(pool_datastack_create(), kl); + random_fill(buffer_append_space_unsafe(keydata, kl), kl); + buffer_set_used_size(keydata, kl); + ptr = keydata->data; + + res = buffer_create_dynamic(default_pool, 256); + + /* store number of public key(s) */ + buffer_append(res, "\1", 1); /* one key for now */ + + /* we can do multiple keys at this point, but do it only once now */ + if (o_stream_encrypt_key_for_pubkey_v2(stream, malg, ptr, kl, stream->pub, res) != 0) { + buffer_free(&res); + return -1; + } + + /* create hash of the key data */ + unsigned char hctx[hash->context_size]; + unsigned char hres[hash->digest_size]; + hash->init(hctx); + hash->loop(hctx, ptr, kl); + hash->result(hctx, hres); + + for(int i = 1; i < 2049; i++) { + uint32_t i_msb = htonl(i); + + hash->init(hctx); + hash->loop(hctx, hres, sizeof(hres)); + hash->loop(hctx, &i_msb, sizeof(i_msb)); + hash->result(hctx, hres); + } + + /* store key data hash */ + val = htonl(sizeof(hres)); + buffer_append(res, &val, 4); + buffer_append(res, hres, sizeof(hres)); + + /* pick up key data that goes into stream */ + stream->key_data_len = res->used; + stream->key_data = buffer_free_without_data(&res); + + /* prime contexts */ + dcrypt_ctx_sym_set_key(stream->ctx_sym, ptr, dcrypt_ctx_sym_get_key_length(stream->ctx_sym)); + ptr += dcrypt_ctx_sym_get_key_length(stream->ctx_sym); + dcrypt_ctx_sym_set_iv(stream->ctx_sym, ptr, dcrypt_ctx_sym_get_iv_length(stream->ctx_sym)); + ptr += dcrypt_ctx_sym_get_iv_length(stream->ctx_sym); + + if ((stream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) { + dcrypt_ctx_hmac_set_key(stream->ctx_mac, ptr, tagsize); + dcrypt_ctx_hmac_init(stream->ctx_mac, &error); + } else if ((stream->flags & IO_STREAM_ENC_INTEGRITY_AEAD) == IO_STREAM_ENC_INTEGRITY_AEAD) { + dcrypt_ctx_sym_set_aad(stream->ctx_sym, ptr, tagsize); + } + + /* clear out private key data */ + safe_memset(buffer_get_modifiable_data(keydata, 0), 0, keydata->used); + + if (!dcrypt_ctx_sym_init(stream->ctx_sym, &error)) { + io_stream_set_error(&stream->ostream.iostream, "Encryption init error: %s", error); + return -1; + } + return 0; +} + +static +ssize_t o_stream_encrypt_sendv(struct ostream_private *stream, + const struct const_iovec *iov, unsigned int iov_count) +{ + struct encrypt_ostream *estream = (struct encrypt_ostream *)stream; + const char *error; + ssize_t ec,total = 0; + + /* not if finalized */ + i_assert(!estream->finalized); + + /* write prefix */ + if (!estream->prefix_written) { + T_BEGIN { + if ((estream->flags & IO_STREAM_ENC_VERSION_1) == IO_STREAM_ENC_VERSION_1) + ec = o_stream_encrypt_send_header_v1(estream); + else + ec = o_stream_encrypt_send_header_v2(estream); + } T_END; + if (ec < 0) { + return -1; + } + } + + /* buffer for encrypted data */ + unsigned char ciphertext[IO_BLOCK_SIZE]; + buffer_t buf; + buffer_create_from_data(&buf, ciphertext, sizeof(ciphertext)); + + /* encrypt & send all blocks of data at max ciphertext buffer's length */ + for(unsigned int i = 0; i < iov_count; i++) { + size_t bl, off = 0, len = iov[i].iov_len; + const unsigned char *ptr = iov[i].iov_base; + while(len > 0) { + buffer_set_used_size(&buf, 0); + /* update can emite twice the size of input */ + bl = I_MIN(sizeof(ciphertext)/2, len); + + if (!dcrypt_ctx_sym_update(estream->ctx_sym, ptr + off, bl, &buf, &error)) { + io_stream_set_error(&stream->iostream, "Encryption failure: %s", error); + return -1; + } + if ((estream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) { + /* update mac */ + if (!dcrypt_ctx_hmac_update(estream->ctx_mac, buf.data, buf.used, &error)) { + io_stream_set_error(&stream->iostream, "MAC failure: %s", error); + return -1; + } + } + + /* hopefully upstream can accomondate */ + if (o_stream_encrypt_send(estream, buf.data, buf.used) < 0) { + return -1; + } + + len -= bl; + off += bl; + total += bl; + } + } + + stream->ostream.offset += total; + return total; +} + +static +int o_stream_encrypt_flush(struct ostream_private *stream) +{ + const char *error; + struct encrypt_ostream *estream = (struct encrypt_ostream *)stream; + + /* if nothing was written, we are done */ + if (!estream->prefix_written) return o_stream_flush(stream->parent); + + i_assert(!estream->finalized); + estream->finalized = TRUE; + + /* acquire last block */ + buffer_t *buf = buffer_create_dynamic(pool_datastack_create(), dcrypt_ctx_sym_get_block_size(estream->ctx_sym)); + if (!dcrypt_ctx_sym_final(estream->ctx_sym, buf, &error)) { + io_stream_set_error(&estream->ostream.iostream, "Encryption failure: %s", error); + return -1; + } + /* sometimes final does not emit anything */ + if (buf->used > 0) { + /* update mac */ + if (((estream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC)) { + if (!dcrypt_ctx_hmac_update(estream->ctx_mac, buf->data, buf->used, &error)) { + io_stream_set_error(&estream->ostream.iostream, "MAC failure: %s", error); + return -1; + } + } + if (o_stream_encrypt_send(estream, buf->data, buf->used) < 0) { + return -1; + } + } + + /* write last mac bytes */ + buffer_set_used_size(buf, 0); + if ((estream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) { + if (!dcrypt_ctx_hmac_final(estream->ctx_mac, buf, &error)) { + io_stream_set_error(&estream->ostream.iostream, "MAC failure: %s", error); + return -1; + } + } else if ((estream->flags & IO_STREAM_ENC_INTEGRITY_AEAD) == IO_STREAM_ENC_INTEGRITY_AEAD) { + dcrypt_ctx_sym_get_tag(estream->ctx_sym, buf); + i_assert(buf->used > 0); + } + if (buf->used > 0 && o_stream_encrypt_send(estream, buf->data, buf->used) < 0) { + return -1; + } + + /* flush parent */ + return o_stream_flush(stream->parent); +} + +static +void o_stream_encrypt_close(struct iostream_private *stream, + bool close_parent) +{ + struct encrypt_ostream *estream = (struct encrypt_ostream *)stream; + if (estream->ctx_sym != NULL && !estream->finalized && + estream->ostream.ostream.stream_errno == 0) + o_stream_encrypt_flush(&estream->ostream); + if (close_parent) { + o_stream_close(estream->ostream.parent); + } +} + +static +void o_stream_encrypt_destroy(struct iostream_private *stream) +{ + struct encrypt_ostream *estream = (struct encrypt_ostream *)stream; + /* release resources */ + if (estream->ctx_sym != NULL) dcrypt_ctx_sym_destroy(&(estream->ctx_sym)); + if (estream->ctx_mac != NULL) dcrypt_ctx_hmac_destroy(&(estream->ctx_mac)); + if (estream->key_data != NULL) i_free(estream->key_data); + if (estream->cipher_oid != NULL) buffer_free(&(estream->cipher_oid)); + if (estream->mac_oid != NULL) buffer_free(&(estream->mac_oid)); + + o_stream_unref(&estream->ostream.parent); +} + +static +int o_stream_encrypt_init(struct encrypt_ostream *estream, const char *algorithm) +{ + const char *error; + char *calg, *malg; + + if ((estream->flags & IO_STREAM_ENC_VERSION_1) == IO_STREAM_ENC_VERSION_1) { + if (!dcrypt_ctx_sym_create("AES-256-CTR", DCRYPT_MODE_ENCRYPT, &(estream->ctx_sym), &error)) { + io_stream_set_error(&estream->ostream.iostream, "Cannot create ostream-encrypt: %s", error); + return -1; + } + estream->flags |= IO_STREAM_ENC_INTEGRITY_NONE; /* disable MAC */ + /* then do keying */ + return o_stream_encrypt_keydata_create_v1(estream); + } else { + calg = t_strdup_noconst(algorithm); + malg = strrchr(calg, '-'); + + if (malg == NULL) { + io_stream_set_error(&estream->ostream.iostream, "Invalid algorithm (must be cipher-mac)"); + return -1; + } + (*malg++) = '\0'; + + if (!dcrypt_ctx_sym_create(calg, DCRYPT_MODE_ENCRYPT, &(estream->ctx_sym), &error)) { + io_stream_set_error(&estream->ostream.iostream, "Cannot create ostream-encrypt: %s", error); + return -1; + } + + /* create cipher and mac context, take note of OIDs */ + estream->cipher_oid = buffer_create_dynamic(default_pool, 12); + estream->block_size = dcrypt_ctx_sym_get_block_size(estream->ctx_sym); + if (!dcrypt_name2oid(calg, estream->cipher_oid, &error)) { + io_stream_set_error(&estream->ostream.iostream, "Cannot create ostream-encrypt: %s", error); + return -1; + } + + /* mac context is optional */ + if ((estream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) { + if (!dcrypt_ctx_hmac_create(malg, &(estream->ctx_mac), &error)) { + io_stream_set_error(&estream->ostream.iostream, "Cannot create ostream-encrypt: %s", error); + return -1; + } + } + + estream->mac_oid = buffer_create_dynamic(default_pool, 12); + if (!dcrypt_name2oid(malg, estream->mac_oid, &error)) { + io_stream_set_error(&estream->ostream.iostream, "Cannot create ostream-encrypt: %s", error); + return -1; + } + + /* MAC algoritm is used for PBKDF2 and keydata hashing */ + return o_stream_encrypt_keydata_create_v2(estream, malg); + } +} + +static +struct encrypt_ostream * +o_stream_create_encrypt_common(enum io_stream_encrypt_flags flags) +{ + struct encrypt_ostream *estream; + + estream = i_new(struct encrypt_ostream, 1); + estream->ostream.sendv = o_stream_encrypt_sendv; + estream->ostream.flush = o_stream_encrypt_flush; + estream->ostream.iostream.close = o_stream_encrypt_close; + estream->ostream.iostream.destroy = o_stream_encrypt_destroy; + + estream->flags = flags; + + return estream; +} + +struct ostream * +o_stream_create_encrypt(struct ostream *output, const char *algorithm, + struct dcrypt_public_key *box_pub, enum io_stream_encrypt_flags flags) +{ + struct encrypt_ostream *estream = o_stream_create_encrypt_common(flags); + int ec; + + estream->pub = box_pub; + + T_BEGIN { + ec = o_stream_encrypt_init(estream, algorithm); + } T_END; + + struct ostream *os = o_stream_create(&estream->ostream, output, + o_stream_get_fd(output)); + + if (ec != 0) { + os->stream_errno = EINVAL; + } + + return os; +} + +struct ostream * +o_stream_create_sym_encrypt(struct ostream *output, struct dcrypt_context_symmetric *ctx) +{ + struct encrypt_ostream *estream = o_stream_create_encrypt_common(IO_STREAM_ENC_INTEGRITY_NONE); + const char *error; + int ec; + + estream->prefix_written = TRUE; + + if (!dcrypt_ctx_sym_init(estream->ctx_sym, &error)) ec = -1; + else ec = 0; + + estream->ctx_sym = ctx; + + struct ostream *os = o_stream_create(&estream->ostream, output, + o_stream_get_fd(output)); + if (ec != 0) { + io_stream_set_error(&estream->ostream.iostream, "Could not initialize stream: %s", error); + os->stream_errno = EINVAL; + } + + return os; +} diff --git a/src/lib-dcrypt/ostream-encrypt.h b/src/lib-dcrypt/ostream-encrypt.h new file mode 100644 index 00000000000..b6f628c2b3e --- /dev/null +++ b/src/lib-dcrypt/ostream-encrypt.h @@ -0,0 +1,38 @@ +#ifndef OSTREAM_ENCRYPT_H +#define OSTREAM_ENCRYPT_H + +struct dcrypt_public_key; +struct dcrypt_context_symmetric; + +/** + * algorithm is in form AES-256-CBC-SHA1, recommended + * AES-256-GCM-SHA256 + * + * Algorithms (both crypto and digest) *MUST* have OID to use it. + * + */ + +enum io_stream_encrypt_flags { + IO_STREAM_ENC_INTEGRITY_HMAC = 0x1, + IO_STREAM_ENC_INTEGRITY_AEAD = 0x2, + IO_STREAM_ENC_INTEGRITY_NONE = 0x4, + IO_STREAM_ENC_VERSION_1 = 0x8, +}; + +struct ostream * +o_stream_create_encrypt(struct ostream *output, + const char *algorithm, + struct dcrypt_public_key *box_pub, + enum io_stream_encrypt_flags flags); + +/* create context for performing encryption with + preset crypto context. do not call ctx_sym_init. + + no header or mac is written, just plain crypto + data. + */ +struct ostream * +o_stream_create_sym_encrypt(struct ostream *output, + struct dcrypt_context_symmetric *ctx); + +#endif diff --git a/src/lib-dcrypt/sample-v1.asc b/src/lib-dcrypt/sample-v1.asc new file mode 100644 index 00000000000..c69fb509546 --- /dev/null +++ b/src/lib-dcrypt/sample-v1.asc @@ -0,0 +1,48 @@ +Q1JZUFRFRAMHAQCtAEMCAMyKuGO/j3TPoXzRJ39THa1oGDChmueqRVFlR3qnIWcd +Mt15zp+juTJzwfxKDNsgdIfFleIbuuo1AX1TgimaVfb8ACB8mhA56i5P7XPoHdP/ +w/oi6kooNSk5rd57+OqFiwD6TwAgWi/IHZ3tFmaohetUkFowgcrYwMh9HR9iOXg6 +QdIDnqMAIIrC9OcLyuMzUp18LpVKZLg6QaJEsjrepBatgkqRDgBKAAD7tnI+Rjjg +rNZ5UHYvjA1xsrEhfbyx8X6Vb+em++p+aE+I92pBrqV/XIeR1er1oNX3nxZwEnL4 +UwhavZOMw7Qna0o4bop4PfK65HqnFTmgaiNDBMdE/CFaxSRlI0PLc0jhqxoEYU/d +0hrtPcpQMq0sCxBbHqcnDF1xAK2hAZU12BH+JoV4bI1k1MMn1xAcXxiVdtSO5NE8 +E5fyaMfvTq2zIZtqQY09arrd0DaQ8o/L2dIV6jQVNZLxbRFWVayoWloT/YVSlHhi +w5YHJetffXO02mllj3Mxr5aIfCmpZbrcWMfrF88ksI6HyQOrTHKS+Y95+VFLbxy4 ++BWFGKV4zUGUwrfEDOpxIAZbBHsABjV82NS2TEZltu/ki3EhlwC8Hy+oyqn+LZN5 +LCmbt/maMI0EJU6cNCUCQM8Rq2Xv7xP/DrC3A0y7gj3pT44dY0dMj76gBApKO4Qw +rLcBgY9qycXHtUwvtg/QZJrb2n2AB7h0+B3LgVm8P12l1KAFS2ykugBWUVJSuUjK +5fjTk4EDIaCm0rhs2hNty9OtBkBuQBolkzxHtqp9/+QhIWJEtNtQEdnwN8DtdlNH +/p69sZvkDhmyqSuByCodwVhkPZf5d/oGVUvE73btlAJcl8NZMXDqgKHfT4U5OF6Y +eSXUIz9oiM2Wy1CVqrA2JdFGQ4Hcbf76IP462+gGefOd62atFfvjGGIb+Okyrmab +jNxn/7wtUw42MXIzoAk+GQsOgo77rH075mILWqp0OuAyRTKmsZTP3FaDb4SxQk3K +pw3N7HiNIiDW1wd5BLJE73qxKr+JC8GLs/s+JpfVb+lxzXAKXpEZTGFd/zphF1Kf +J+aBCF+UB5Lq+QYoHfKSJzRb24PScAVs1VrV8nJlQvxMZW6Rd4ofNcsMjY+b1pBx +bv74+oACEXC/F1jpp3tMTTLOLN8/hNq0J1mE3uOYRAaNK1jaQQAoa4rNxeY/2vBw +mod8/Erc9M0M4B3VHVfRJ4F8MAx3b8if3oiicLu0OJ16snGHM7BGp2pOEPRVAg1Q ++70zDctuSV6HwPYSAXaIw2LPrbsQmq1Lq/JivHYRwjUDBFAaXkPv2WFEfUsXEtmw +UzUPLgWxZM8MsPDEI510SzRd3OlBUjuS96+/9n/Qv1D5DBnrDH9gUwUiPr9V+rMM +kmNaAQpTKpCNpevCH/F18rQnUi/PJKY37q8pF/lO0OW7mzS+CxCvFr+aWxZ2kV84 +GwUrUuSdHa1Z/0BY/UTZn1BAi6bOudiWHH5loJBldreSJ2K2/iwMpLUUEuQ+TJJ8 +BvsWeOLMqEpDIxuuYERCUqHO8EmtvsSPKcIeS7ZZpvkRlRIWCuKgnaHSymT87Eq1 +SJskTkAWd1iGvQI78tM8KwT2KDmf7Qs3RUCiSynT/H1OmVQqwVn/5c9wFjV+0PRA +KFZZDEMvwy5tYpJ1Y0nYuUUMOlA8l11rpKqAIcRj0V256uXoj5cBnTsGZAOFDBqx +pBLmGFgz1havj8RsgqtJfCkkh+Y8l9xszAC98/FGYOtmKpP4sXXM3LASkhi2FU4C +OH/4tUsoh3tMCendx36s1UmliE94BiNuJMUAqSh9cLCnW/Uiw0bzqV8GOJLDk/A3 +XMWrERuA2Jn7qQ9e9qmYarc8r6JjWUuxECZ04d12wNCcDF7hWxFYLbwTk/ZUIM5D +1ZsdOPUQDf7gjEW4gQoOK7pQWifJa56ZSSFurLoL5ae+3dPwUu8HNQLvwmx4paSe +01shQd36MXcRZ7BRVp0GqNrviuAtXacSxx1GIO9rh7RtyGGs1grsQ07P6Evpk0k/ +1WY48cE+xWU5SH4JwxMZ3vbDguMY/cnp2VhuzZguJ4iIFKg5RMVShrSkZQcWwH7e +7JVu7hOe3bWp1KeVG41IsOFpo0Jfpegtgf4r1hYih02Q54UNIFf5G3IRsbC1pjtj +ALYteCLe9oa+7lIAVDWgmq/NqWLsi4dtlz97TG8XApIFZ6Prr7KG8N+RFTmouXYT +QH6FuF5XvJ0TqIgIkdSzbuaNmN47E6PQoDAuRJ4X9ahYpF1xC4ecnAaI+rlaparF +Z1OOtwYVQ1Hthc7wp+205aK6ujZyU6a9MrGXGZklRQdigCd+EOs0kWy8t+bcmk6K +9aXEJyGMBtcgQDGWZZJer5w8aUKj6SDp9y7X1kyAuhh8DFvNKMgR3vs4wnvsjr1W +cYxH8RvAVXj76Xjvvgeg3jcJbcr0mPwB0SrcpQ+88mF64x+nrKsXmz9E8wBgOMY6 +4qXXb27YUehAGN81jTZRN8lShl20fnPZKd5U1sqIhevXNUrTVjxVNff0fHRnOOb9 +sVD3vdQmfbXvB3nyQLKQ+Wcw/WPqa0But5KEXFjnaXPORcEWEpYvWOgGPegmrTgh +P4ZVscRdxozvd0sKvpLNMd/EiNLBaYft+4Yo38WZYkZzNJT7LhW41pbQyK1cmZiU +COisSP2rrus6iKCNwaGPMZJNCCdQN862DSTaa0IZSEeSRpfBfA2UevSsfX0uTgyb +B4Az1u4QJv9fQp8oAnnpH7YHhW43V/YTVuwLLF7pqSl2J2gNcLFfGuJVftRp6xJg +mkfXOxoJ0y5CQU7pwQZ/F6WxuWBX1m7MNE/OfI9l9ODj0uAgKn1e+DzH1VXZi8ls +lenQuepkt7zqXG5dRRhW+FSlJy1oo9oPcf2bZhAYMty3mh9F4Ils3SSCE6T6cU0A +yYCNrq8iKyq5V1jdC9haN5NmF9yNiIxYWdbigcBTzR5+AZuNe7aJSXu0qvQJR9s0 +d/l7J6LsH25I/zsIV/0OHcrvMUEf2WzU3Q== diff --git a/src/lib-dcrypt/sample-v1_short.asc b/src/lib-dcrypt/sample-v1_short.asc new file mode 100644 index 00000000000..4bc4c06d2a7 --- /dev/null +++ b/src/lib-dcrypt/sample-v1_short.asc @@ -0,0 +1,4 @@ +Q1JZUFRFRAMHAQCtAEMDAA+5INzRNr5OAicv3XI4jh//ufjZN9yYr7mElHKNGY+D +D2ziqHhPKVra6JzBzZvfySntDnDvdLAomafpDVlESMlkACB8mhA56i5P7XPoHdP/ +w/oi6kooNSk5rd57+OqFiwD6TwAgu4C6eZWV+Spojlk8eOAw784ySgMHK8gDrXhA +Jwg34GcAIPX4RmqXh+7QTAQWtGNHQvjqSygBxSUYkQ3KfPLMymKvAACMLy6/ diff --git a/src/lib-dcrypt/sample-v2.asc b/src/lib-dcrypt/sample-v2.asc new file mode 100644 index 00000000000..0f8adaaa4b2 --- /dev/null +++ b/src/lib-dcrypt/sample-v2.asc @@ -0,0 +1,287 @@ +Q1JZUFRFRAMHAgAAAAIAAADPBglghkgBZQMEAS4GCWCGSAFlAwQCAQAACAAAAACf +AQKrE9JRl23tq1RrZzVOdniCF0DdU0t0nChX9mv2K7rd/QAAACEDcgFCuwpazn7Y +8fjHjMbv0F5XaD+Ao/EO8Kr94qRnDGcAAAAwWOKPJo4WPEpZE1jkw8mohu6pHH9z +O5rLEHGl/NHsavWmhze9ALoSenB6DMisiDl2AAAAIB1obuueyEHz86CqYIOSEZr6 +8eAONYY78lbL+at9dqYOD6rKgiDsWGLdpTAN7k1X6kxOvpCKrFgW3BM6eKKsj7cp +YHp8vANHWohrwXV5+yL7d9Wkeg97pcs5DXAFkqD6Q1KBHPfdM8V8hJoKdUODXq5y +REhT0izOcMjkQSUV4CDr0gz9TiVylUI2tg/ZfctLJNIxKoxEREC+J3MS+xSRm4Y8 +rcOwR0bV24MkclRy94C+BHdS74/Eynr2gBpdbbe4usjttCSjSVeP9ae5ZHxloorm +8v5rOlNhYk6IaQhtlRSRCEqGrfeTDBi6RyzmdNw+E8+EpoizKtCDbyRPLEX8iSsM +rY5ANtrFR3QGxbx+lOeRPBM+Te9Gw0j1mKvx9POBuSx4QJ8ceTPa4xJD7k1haclX +GsYtJLxSnrs+vMhWqGBcea2hd0ZepTeBWmv4o/SfHrgFzfH4Tfc+9EhDx9azP1uZ +kEny90yefLues4oeBkv3PIJ9ZkN9tMImjPfiphZfYBID1zJDI6tvqW34usrlDNIg +ptLyw1qwdyh34v6x9QcAgSOnmV0lc6mrNLt8VZqQBNTdFQFVuRN2Lof25WUCHwzc +1kZ64jHdbu/4ebrb6RR5dzXTTBC1oYu0jeCubKjtWfu99e6hfSZ/28VUZsCA0SP9 +IT8IYlUcUlTNbpwU3ZAuHUvJitXBTatdss8mrROhRLqT7nKxL/+W+dgC4aihIh3+ +X3VarEf5XH8u2sBz9CjUiAR20eXve0C8Je+jOOs/ni6lZIZh8G7Tkm+GE9rAcPXu +z7U2opxQuuFNGDI6b67XyBTv5Z1V45P2amMPOq68IuDOH0moT1yNLu1EkmKvGmFc +MNBCaKWWTj6ncPpi0AtFa8wdZETNyCK5Ljh4wiSqEQb6LgUtH0XJ34ypyFUhEbr+ +Jue+5TulZh0O4ltEMXJniLkoZPiEDS2z9CDH0DAVwLc78UZHvG/DcGpljew0t16X ++xUciM5oG/9qc2m/h/sDga7Fs8WVe65iibxxB68xgXjs1P6dlVQ6ssdrqKcNz1jg +/F/cPX+aWaPXKsFh5rJveQ5RzCCzZxGdcVC/nVJZBgIk3CuCy97LscwRhVmhqSx2 +JRXfVMHVy0t2elNnaWIR7wyhTUc2B2qZkPiWw1A7TRdbhNQove1S1St9B/3DATty +s4XxhBBFYkiKDbcLDwXHUN3HOrjAUR+4mcSOHVLnotuHjg6KEOJLxrFrdEEvQ1CS +OLJwmJ6ulIz9aPuhVEQKXH3qgQZFlmZyukH0x8qfRtJ5+82DQ0SbZUENcVjwDZwt +CCBVtHJD2H4pC1KkluXrYaY+ReWwVXFBuyL+UqJp6cg8+O8ft/y8e44LA+tdojK/ +hbEuMramyuJkhGsVuR0Y925A/j+hKpefISWqGVgWJEQ1Ry0Z7cFJo3ApNjuyd+LF +5WW0jCZZVehvvtso29qSrTq9zH6QiiJBYd40VzLA2TlXTkPFJJN/aorkRysyfHX8 ++2Gl4yqC/IwTIxupLgzVQd+9g5922e8VKDRV2T2x1AbcmObr4h8FwFv3FvB3Aqvj +kKDQk7u+dXN23+OxETQ0d1Rcdr7HrIY/QL47/v1UiSVKQKUp+6RK6iX34qOnkWBh +oXtvFuUmN01yNXCLm+QuNbDo8ESvgpBGAzVOtUoSfIQRIbjcONQvW17PJOnRKKcx +bpYVImf3EqZnZG9gqUaoUYed40kW0RsCH/PCYaejfnByYyrvPoHNm9a3us1NFZ3V +g17Hy3LE5BwrutFsMHfU2fnFQBXAnOzmGGQarSUSyaLprQed2qH6hE848zn035q+ +IMeqTNA+bjroL3R6/IXmkstihLlD5FMts2uCm6pa3QKInucRhftscgO87cFqQ6uE +sxryFRLk3Ij2M21OWfo30/MaNGWZ6qeXrss8wvrlMfloxc2ERpHXE3M9ocQhWzpK +KMSTojQjO5CxZU6pu/UmxALziCm3+lPQwvfbwGcC9ewON+vWmxPuEsEW0hAYBH5o +7j4zEJ8JMn6HlFEDUcf/fI1YsfV9cJScmQSux1IGh1sr4rHSk9pKXCwyf0U5ZrAA +KTDxuA8BVJLk/3nsoyYSMOm9vNVrgCnZ0YgMfHtwa8Nrcz91VhJ028FEmhtA+bn6 +TJeODt+RYY89XOtkhS6D03y7n4DiBa2TuWevnXA0wdhOXimmgnx5tdWKDtQQbRvV +bthbvuAwqWW+ccrcyhkI+2uvDpTO2XAvcnoOYlHqN04rMo7TlOiVmC0/MGXiLvkF +dLWFuZ67V/l7UWCLDKG0pwweJefd2b37gls/mpuQxK7OY/k7CEjdaPLdrNxlBNke +1VnATpDi2cflx6lm2FF6XsCyQszxifuQGRtQoiFgZrC1ipalVpeUs2LHKaQid1hx +1e0yaqmV+zPObh7q4y7wxoTLGb41ABI2jT+bSYB64Tca3qPSU19+xouLJ9fNGEkN +Bc1kFoW/ByYHRn6LSChwazjPhiNpuw/vbvmeeQ+NgAljVmoCwGcE56ACjw5b1OY4 +KcoyuiNTkIA9CkwgQbvdj6dyNncitBXAvW9RHlKIMlhTC1l/B6WS1FscPmdZOteL +18CW2ndKwHcqCq/CL17z4cepThmcMKIHvI1NUde31HHFY8A5hNLrybHIYtdTzUW7 +KQDJQWzSZ40c/9/DWON/6POrhRaEnwexcTkPopk0ZxTrqPUjYvtQFkuiiHop3YBL +Bq3fnoym94FSqg+yEjtEeDBRB3xMuZW4pFek6tp6CtaqfXrhRALPEc5ot3sEOGPf +Oy0oRqn4bwXwWtE/xbRRu+bIwD76gwNrW06w45beWnr49xtO5WXFmoiahgGvdE9Q ++Huy5XugIJ1ycVXNbkLemIPFfRBtUCnYWb2pX+tfGLKRg2HHvXnWXYNBGaPdoJV0 +RLamQMELYM7OHbhbcIcPIMpdx6DTSjnf72oLH3oh5DZAi+XmuXbbJoWVmMm8mHTr +995HqeB3X19drYI/1zkTgzIzXN96tYTryOWel9+EsF2bTk+0Xs02bmr3NergMUaX +VV8+t+5RQoJhD+C7WARJ0wVZkQEsuR3OL2rHWHQaUKEtuJ/09Hq45hYFTYeolyp2 +Sjj9fEuFLaQaKppid/J80OOaffcWo0By7zfyrntEOcYY/KqU0cd9GH/P1DiS1t1m +5f852ONOYKR9nLSJAMuQkFefrCQjb3y1cJ1bTXtGM+ECvterwbi7V3PSj4YHBvQC +0r/NZcXOVbDF+MEwyqFpQqT0XYiATDtGRlqfyxDyG1Sz6+5S3/8Le8q78ONko4jO +pS0BPTU9OhhY2Ny4ZLuuoyDqVHdv1acmzP0Qs9iN4H741br98YL2HJQU10nQHBvZ +fJ7j9SXov1gtNmhu4HXTYDY5wBr+xJYy4VGaif0XXDCv0G41gdo1xH+H0tV+MlKT +0BesyEFMGOkx4ROUXQf603XbusND1CNMXeZVLximwoc6aZhFV9ujyf1hoNbC/S7+ +VhTH3B8QuLDQjj+GtS/lg/MtcnWCgO+UGx1jctUI6efab49SRdgupcEw4NS+ABc5 +mgSAKTf/A4B7AdnPhzr38GFIpTKvygiXUxBZVJo9c4tOWJCK+ECuspUtez9HTUUg +QR8yAewrJlseEUcuSuJi3y8gLBB3XlkPXqWN8bORtxHgOPcJxE8D7dU9zKFM3g66 +zI/5PNAp51E86d56AFqTs8jbSXSV3OajZSAeCYDnhowv8wI24+7Y9kgDiFmm0D7i +AuoGxOtaHI6p8lFtwZ2FHGt+VLcUhPdbUNpticqYkNasp0G2I2+cT6FDLl2C6Bj0 +UpprQpskkugoNzGDhAJmTp16fyj9P4AFwhgTxOs62TUE8nUxeQs2vkftzJnYKUFi +x9RynvKdBLsGUi+ZUOJOk9y2xgeNvFkDdNNWGa2wxI3bHQZkU6d8SNHRwUwWuVPR +C65sTM0ky4wUS66eD49cdaH/8uFJl59yYnpfwVpLp6wLxPilSzqLkfFgMeXPTQUS +7B2PO+6FQMKJOtbyk1OraFELz+lhpfJERpckPvjf0Wa9o6ZJ1z+6ZDPCxB/eSGbn +iFbblggiHmhGIfuQbpZhyaBUG4y1U3jOY07OEHmOpuiFu88Z6rwPvTazq7qw9I6T +tamkHEJtU9K8gOYLacnYBiThbpw/z5+s0v3WLqVibuH//R1jBG5PCFTQ8JyGvlr+ +xxS+PdtESXz9WyNG9fX3LPIP2mny557zIRDdXzJLnxlnlPj5d+TQBgAo9kHtkmEU +EE42UzOu2hpObM9MTQ1v+UpZ6BFd2FyIgLdNx7/4SKRsp+yT0Kf/pM7w9HVNopFu +zYBKDlNuyeZiZIBhNtHT3m1aKy8InWTWgYM26zv2tu18ay0XlLYfdvvlCa96+dT3 +UfOMVIMw4FqTQ4B3hXtk1Oq2fNCdWC/eyKBsGcXckpGwRDJvbTpDOapFWB4bDpMd +UOM1acfS3BPIYNvhTmfi+nTbLBM4oMSrQL94A4xojz6IVTPH7KzKS9/f1ZrhuQ+H +3wmJP3w2oBzczMS6949lPl0zcOd1uufi9idoSGKqvCm1XuC9AZzUw+61IfYniuIX +xYFsh1XhTMKazeJtoXwpET5pbAwGCjWLopgwQ/tM+/te2LVuxwETN1LU/1EYm9wC +Q/ETT/bNClrsUQo+4qyfEV2CIA5STGYggP1TXIBJ003IPNLcgYp8asBMoHkrAyOl +0+irbL050ae9xSpc7voBFhNIAt9NaTBNxZ7AtGblsNy9pxes4Ql0oQIC8v+ehddw +TgeqcnUAORcK0VEjQZ8OW0w4a8jRYxD8YpLyGb/xKnis1O+2oFAew7lghN03kGVr +C6qn8HtsIpGU0tmCQVovhVdzFDO6noRYuPFS3L/WrXBoxAQXValj4fIGLhDrtbiV +u6N8HlXpgk4PLwjLjtbcOBUDcPJoZTWgpH9A/BNKpjpO4vDOwoe2vtRxG9f6v65z +xRGJKK3HchSeR4BUDxi2/9PaAW4MgA8jZvpBRTbyzcfMKQyg+z71ijbAuO+3ghkS +xpaVByl9VFK2F0CppdNYeTvPdKWnUNpPHtTfxYHclHLz9xQg392KORC9kWHp7wTo +usm6df9pwvurKtcKa/QzBofD0wAktMVPjwwxT5lWulzcjBmV7IyFyeXob0zTfys5 +6Up9z7tDwkzUegQwbgW8dIJbuA8krqW0ipZcXBGUrWyIiGi45kqybPAatiLQPB++ +3HQU5tfruBEnLmkIOwO+aQcdwrBs5cv/kgHsFfPyQgdxgIDElN+Wl8UvYKDQgnWw +3uLz+EhywwjQJLX5cFMI8mHVu8db5LTO8ar7rrJ6XviYsLVDupGIJjNixCW+4dKE +qm91B4X2mDLZlaEAsVSfPa8pDU4BE3G14fWb1qRd3VsOF/NpJUZ11m8Du6jskScS +B3+xlmN6Zs4T6pE+jBr062Cmma2Ngt0B/VRAsWSU8fUlvGdrNfkWGJDz1aA6xZ2m +LLh/x2SKNh9kP/PRcO/1skO5OaoSYiPBDfSehA+4L4mMpOTpkhl0R/VsIyvQVcEX +mWFAXH7DqL5WB7XhIh5cwzdBkIqXVsTinlDHujD0YvzayaLDXKHPr2riTocXd4FR +CWe/hk9r5ykGeRyYpXGwdLXh9uCv1IMQUw6KHn8VMs8fJp5Wafaf/JYgZWkPOY3c +jlYbW3bOaaYUFNr6jbD5jl3WqAOjXRlmx4wTIcsqxzRnjudm0pHF4G6MSNTixITc +He2QnYveHeUtZNS1nTEdlMqXzrJbbypOcZ6QTrjjLdltr5dJeAni0yKoa2SISgdh +bNztr6pjqqDsUWUh3MYZIANLS5XRBmlKluP6gODFjc2M9JQNUnrHYTVj9raG0t4y +6VjDf2qmIxQ+C5qBiULdufFOqg2tuo5KZeka3HHw+g9cPenXR+Hy7wYBhrcp72Qg +5+uPTpOIdwNNNnBCUgalaaUI8k4rIkTCUImrv8ATvkRfeCn/FV8UubHnP+v3WMfM +Q8xGQGtmUCk7HLXChvFs5JN5zSNRPMczB+tlGOAZg3ej+BNzy2cqHGtptBKq/ci8 +Qu41eOFM5FsAS4VdYmzRQXeMt2AvfU+33AWAZBqkGcapyY+X9iWwPXBixTYWYW64 +y5b3yv85ptzelY7tQQiVNtAW3xLdprjIuwjxgSoAaIjtfORnrtrEp75ZH9Ojgna8 +ad/w/1R6UL+tHHLBvTfoCFiAUjHhi3ilpG1c70d7j3YzO7431+KAkw5ssv/MgtIs +aY/PL3X8scoCRRfKs5TTeIoqPEDq/GMSX8wY6y897DOLxLDTFuKu5T0F3lgB16i9 +0Ly9YY/t1tUetNFg7elHVoLfa1KJHlnZs0smdKRhIcnmMnVcJ3lOhyIlXP5ukW4X +BRCMmdQJ++G73OSzlBrjg5L4uW4Ct2raN38GpE9thXWkeu0jxfq+urRwbm47ZyS3 +J/54NS2hJc+BpvR5YIwV/7Ss7IC7z4LmkWo/WS0xVxSK3DV6Kph3k5v6paHrJjx7 +8z9mva8qoBLTUD0tX3TkzBA/Scv8wbZy/6F2fOt3PGCXey8K2r4DySti44JEZNdr +1LJraB5+bcIBponzl89oEmeBq67pQo3pJzwC9gqnv1xjtSvcVoIanxA3zc3Mvu5t +VmQgAA4S7bnIL/Gya7tBOVzGT/tvbAYpDzDRstZ4xJFEWBJbSUz4KM/gaU+l92hb +2VDflKjvYJvyhSZK8kserxenaya4BEB8EC4T9QFwTNPtU5pxX/Xk39RMxX0KwwwL +nadaU6UodsZHa1SDapCz++U5DlYZi8O05rywQOvuSQ/egGBHbxbU3zk/cE8W7q+q +VHLE5MrNa52rxHGbssEHDzv8ppS+oVpxNs/OruaMIbo78s3CoCfer/AqDjp0t3e0 +T7lMXAUtJ9CWIX9lrCUUNmr1JIria1tDN943NmC9hwioQzvnnzryH4jksKWCjb/b +bnqcSuYe8jpfeu5M3kLiyRKFje3+hqjhR1rIQodWlWS924Hfrv+KXs+gciiwfH2L +wHx9UOx6mDXgJKs4XJR+MDC+fJP71IsENASak1THqAb7lll+KBJgL0O9J0zkztcE +cPe3KxYMNYO6jYnhAAAFOwQsRD3Q8/G1pwif7HRM5PLb31/GpPubq64lVIdbR2rn ++PCztxsS0wRQgYUGHtr+o1SEVQoNZpNKBnmjoV7SGCATIAz4cb4jfmbC0g8ha2+N +HbtL0G6rU3OXLwjGB1CguZtZyuvORDCX5tizilh7m7NPv8QtyhIMlaS+sjFymCdf +ve3js7t/Ar458vlSgWbFUgSFFsqneomW6wJg6aF3DmoO32P22gGsgE+Tu//zVxS3 +fgoKrPKF9DPKvBm7/HrKYfJFONbNIg7dA4/SZWtMYkFXA/R+37+jQWN66kZ8mpTk +Ix2aU8uW8/M2oWNNsaN/T3N5oBiEuPEBjf3vKNul7xB7VgmbRZm5Lom81gFRc400 +9kwfeQ5SRwyaw2VtfqnREHJvmYUwLjEq+uc4IfUduAZFPslmV9+saI9Kdx0Aw1Nz +sHSSUgqx6ZsySW3WqQt7ed16MzLX7Zks3r+skoTwCq8RppnwHQJUy/KtRUXuap+c +FqZcPM3hPjRl2WaygKzEPd9j14+V9O/zkWk1eLwlD5Wx/Ou2vAZhP45yqzGamEpo +HXthWnll0dMQ7G/ye0hxxbU4dx765w4bVClBY+kKcCpSR6PTpo7CdKoW8sLiqoyy +Yo4SPZZzsfhbR/dxchJHsGrV/wwH28YTLNsB/aEqLDYxEy0TN3K5Y97Ot4LNeu3J +7E8ES5PrZNAYFTn0OapgE5lQm05Otvuul34tVt/5SmO0UlaVjOZP038XAYnn+8BG +eq8meCHwnd+2++UfAwuobKU9NPZ2I87/I3mPBsYGxgGQLQiuPlvNyH8yNNnLl5BN +0+lSsdjCi8Z9tpIdAfu+EUe2GmJ8DeJvZbUBDXBL/QGqglq3kl1CewnS5vsq9+zq +g+cAk4BQ0W/4xKFNsJA1F5ZDD34QSwfKUZgeVnRJp3S40rGz5/9Hap7YggPCFSQH +g2SWF1/xpM+Whi8+YKVRYRYTS0n4LXgDWSh8don7/dYqUZhGXD5WleLrdX/aZekp +19TaAGgeMREyJT+nXOszmYNJ8/HetwDhpD1wmlkN8GZoe9XRrk2A5hhSVi4PWqRb +YUowGF93Qhi7iTfoJKJahGJN8t3aDz0WPKJFvEaOKVMH12crZWGsdUylzkSf3Ekh +qTp5Ognq9F2aqJdkfbT6eI6Dp7gruECPnm59UTvhiSxXQZVjy9CMGdHOrCLtqtGF +xYN4Z2CYGg1EKF9vyN/YCkJgxUNXnuWPYie3vO3kUNIa/NGID6vfkiLXhqRZtYHg +wJRafUklWLiXjFQ+huuQo/+ptyAu/3WyP+r5b9Y4F4FqKdjYgI0XM1N9TvIUw25G +9vRZmke3aU3jDdxZfn2ecp2QaHmkBRqp8F/nyoM4S10C8Kv48XwqHXyfxtSSY9NF +kKH1WSI463C8kNwT7dzKdOgI2Jmgj2fY+gznDTJiQAag4h0gZrSllMjfC52x0NMd +y8E+MxBKAsrJS0bUxp1o4pcFq0/4HCFDC4KpO4VfzP4RuVbR68TOZesWLMJ4TfAW +QSA2gOIcPeoyXA1Syfqeuh5w012sbaVAKb8lcVWXJqAayeOj1G+azwAof88mFw8V +VwEFcHDWK6c1cJKirz7C9XyWOBWMOYYzX3Ul9dayqwp9sc2fskgYZqDQ0Z6TUIuv +TZ/2yQo8RkYyMl/Z3Zmp+LaPr1n+hEz07TSEHZUObE45KtCpA7DxxX/gtgyPICwZ +Xh7zaVZlH9TlGXqD7EZDzI98hVsHtkfH0Gw3igYaFXJQrTW0OmBBobTCY2eExHRj +nVvtPMIf1T2g7BhWk3kZha4golyooD9Q3NVlrot+xv5IG2kPXJtMtbwH6yp33K1S +aZus3jx44Af+sSaJvK1RdI6LGGe5gaVnURxSllQjxaN7x/XsLQE8SoUFGaSfXm5b +lgzB48lDdutYOi68jm7yKaHP8Bk/Xjm7UZ1Mpn2ZHcYf8x3+o26oRBgTqb3TAXkh +7qpXUArBKIRb8KaLOhwQveelyMeFsen30FQGrfSs5aOeMsD0IqlDKq+XU6Wakmv9 +2s9xkzI3Urxw0JPL9sRqq8hSY8w6Ae2y6haODvOcWva68w+aUeDt3rPcIGkQb4Qg +X8oRzP/8GpNyXhRCIvo3GKAOcr4Ls7qzOtDO166pGGf40Hft8ZT/roBVbcrZSLkO +Ijm8/hvuOYpQncOaIfKSzmjw8nBDv45nWeJBqiviARUlJHtnyUYLhULwVObcDXQE +sn4nwHYw6dDUtMU5uVFjsR+5VjOXbPWhspZ6AGy2csfAjsvlEaxEB2/15n5pmDOE +oOT0lqmZ2AJHkcqtL1Fqs1Hgma5Bey9+4jXc1CN5UVR23K5XY/tv3btj9BxaQizR +W4CX0TR/s75JX93kbhHZWvT0mRYqonx3+8TdR7IOBvHsGeVG3lGK1v0OHWxL1KsZ +WSAlEuSJXNlWMH85X30zjb9PGcUItpMapkBpUhlSPF+9G8GH0Bv5UXyDdVjwWNzp +6BMSdlxhgLiHn5AdtSIWs/dgleRPV9v2BAzqVC8vqWiCmYbY+HYUkBE/H9qpToEk +HDeKoSc+PWyHCE2McxZWaIK9BqcEaqhoUOryBaeBkLR5w3EL+q/KCjYvNK3TZFUA +OnvhzBj74Mr4A+0wfwW8769JnVeMw58orcJeOvTe7SVjGpso7ZOVAUeyRed0iN6n +kO33HFv2zZEBPpEZd164ue7vfHpqHBAKwtFc8bpBjYLNWqTnBHniYuMv03Kwjc0j +LN868aBfTxU3dftdPNHh//H/q4XnmTWHYYmQH+IJ9zB92GIj8quiWjI3fmxDmWMN +H67PLOPWM+8LJ4vnfMsbEbpYpdioMd5eNckv2Ip8N6/t92HaPRyKZeRH2Nyc+inq +H2EJVXfrdKtGtDpENDsJTrL2uOqGLNb5X6YUOuTEvoBwHx+cw57ObB5lm3cUw7P+ +Zo6/+qOrGjuKOS3NPt4cAk12UOa3bICS0Z05a+7RzFkXcPQir6J+mWxDzFVBV2AL ++wjMj92P3xFyHIQT4t2RFqZHxB/aQXAQxPSyeHUAISOywwwKBve2o2Au8JmjRdKL +IrNFzPhJjKkPoq6ruKjzVXHYjwCDUZ7FLA1+T21WyjBpTm2iv26UNE8h2R1agE1i +E4ybOl8LZXnUt0LAKd/MbrATX6sjC/njj7euG8i0pm+soyykUOi7SCkaHlZc6MvP +Qj13J19Gd26xq7WRcyJ1o1jnXnYrpqDficRt44CsQXxm3rA7KsS1wWLSHysdRw5g +O+fH1Wh8Gz+JvL8LVA+b2oRb/4zbNLgJ2nZOREAq7xM4knZlyVYG5XD5Rne2vSYO +5YJvvGE5/eK3poRc5UoO99fX5KLL8aU4+gpBfDKVUI7FfJ/SqfXcRwwc3hs3/7ed +nMP7JQAj6DbXq3gZ7rCsPQiTpQn6ihkytvehfhbZ33qBGmZARGCX/D/Y1qtLQskW +GJUq8Ce2urCAj4CKtQsxUnOGIZo8DKU51PMNUvtkCTZE61GI2tIHqP8NuN+fDbcf +obo67V2D2pX1wu4aVDlXMneCKF3q6OativUXnqq3oqVemWQgvxjIFhwCfs70YRou +LbKo7IcohFAdb6kIaNKSZLJIhPdGC4pj8PmPUKdKSnjb8IKPxmweJAn2ti6Q2Z5x +EwLo/uC+uYXhEtT7nJ1ifBATxHtpLatjOm6sMnfpVzJPsCPWixs/nP/cQ64GAnnJ +mfhlKHT7ieMs4SNYIGiDIupaSFdrBr/kKS6EHa0EPA1RYwz3a8g4+fcVjOgwpiTi +V0SH9JHL6z2trV/voYSZF1+tsekBQhwRhZcfVnJ+N81f/4k80cJE9F/UHNd4JMCE +f9Ki62USX1F8y8fQdSV/7mSHnjnnI5nD3zFbUY7lnPE+0cyzPUSVAwewTy08CaUc +kP/fqCx5puwIJpMr4yqb3Xt7e5QHTIbDxxnXUhrqf+WQ9w2pWD+GbSA6RHRplqFi +jO3QRwH34yUFAkxUIsjBDGf445LAPy9u4usuoxw3f5/b95HVhn2GEtqj5AfT1vDV +z0/ZlbVZKVjIeFjkXXGSRVNqnS7WFLZ8Yf1Y0jpG/o1H6yhrXQRdyra22sMXGozU +5kBEiHdjqqTTHNqQh7Avw3QKOyaIHy9mDlNoJ6jzeUYOg3+ti1EAnmV4y/6OV59m +uso0zQvZo31UPxpBucE0IIbyfzSG2FFiGhwUxo6uCFxaJZsoVea3swziNR2neNe3 +H+E+dKj/v8cf8Yz6L1NrkzEjIvOMQ2la6soRLztZ1v1lZqTFMFYNWM/F4Oj9dtiX +5KanHFCAOxFtyXkxZ51DzJiM3kjutgBofKv1UEtoZsSqlaYgl7VVfuNI0eYiAhvO +9LQquZ47y0L95g+ihPChpYkrEMqIEv5shjKmCp3iZPITgHO0puezZRv3x8v19Mb6 +gSStswX9Ppz9ZG4eMp8WabiuRIu/D+Em9Ce+26PCGcEkH4Ge2VQQ+2CHcZcpzpiU +8wUqQzGNX43I8sIfNByDW+8JF+799qLQHhKH9x9zQ9CHENXSUamlX7SYSSVrtxJk ++76YVJiqDsoM0zWcyJyDg6Wcr/0VqszNFZdC3SgJcgYxc8LFLlPb14orly+aop/0 +rbErcGnCtBfQqVBmLvd/Anxv3v/BKGbqbELCEg4q+f6OCYY3PkQKMxDUhbBIzGbZ +lCEnDsnMzbAL9qd9FLEzI8DGSbhM2b/qjRnjTk4qsllxLxg/SlS9WASnmZNgK4RB +yvYL1Y7WUc0fPqSgfkjS4KPNgwCxpM+FJ5EI93FCR17u1GDIhI8GkNBj88Rpif3P +grC7a4eAJY+jp7NAedOoJ91r1VKbcY1MaFcBHL25XK/nYDPUkmu7PoWPJtY3kmZ4 +xIPYQJLS2UXRcttvjvZo0+ZPlsWZUUDxAFW/90IQOcHgO3StevV795MFA8pmA3wi +NlH4KlC75aBW5zf1vb2UP7NdeCI5FgmACwoqZqTX16aCK9mNxDHlHz7S661z+FIN +Tea843EHLdvGVwkFfz5pO8ip05VrIz28P4ASPWzf0iys9Ev86TzyovMwOlK1AS8Z +hVb6/Tup3jgPxgMNfuzRT9QSb2OCi/043CFvTqBveGxSTMdfefpZguAmxSv8U7GJ +g2+BA4dydIixbuGPQlbJWWSvUfjWBDUFEPKhDvS9P1YU/AkJMfNLCC/lcYc447yI +1m/L2NNoCtxYicHhSglVM1MB7fx/8gwrg5cgRNusl9as3y5N8QLxFeh4hkYiKZX2 +IXXLtv29fn5gJbpXR1CD+lzoKe8fWl9UHGxPuW6Mv7lyUBKn7I37zyyDC/M1MDdM +5otfsQM2FkOXFSu6rfCa2LWCUyuuLYcwCXa4kt9pRc48eo+b9Pd0OjebDk4q7Ead +RTvPPAXNJHR9GsYxIcaDOI/KyfjKZpnTVOFTgk5qMcRfZDTlg+2FTOcQdhrKWNXl +/LRW4yQHN416q6xDvWK5Wu5P82I9+nU2nWebBKXrqvMvEqs/F3BmnJDC5e2HBF/i +m2XZPM/QttDoxF/DHDhRzXmeBLS6i4F4nc4sCBCTM5prVE52cQXEgdFfL+0rRUxJ +dVTJ0sJCxxc3nmymeHe2ur2MPXvsw9eaQkjuSJyCZJgMtYT9Jr+6q0y62x9Qiljq +QxQ/cerfK2Ggu92CTje5aLWUa/w7rVuYHHp0cVUn+cEJlCn7podgU0KXenEF6Lqz +heju04sJcJXTQKcT7yqMW0EJeIPbCZLrrvJ2bD+dPAAmdL2K3zbQUP+84ZbKVT96 +4BdERkqZiQIRyYqKV7S9nBh6rcpv8vCR7kSMHBD83eSLCZq81JzD76IRo/LH1vxc +FoypT133cch0XR1E7CK036zQTMOMiwA7LrOaNYi1lvbBW1HGGqMkv4pvh3Xfhese +toJkyd2Ev5xFmRMbVQ2ABrO+Gh0+Q6aeMoF43U+/TeytLOT6K6hPVirp8vRMsZHS +cw/JBx8feWm5eyH6CK6euF/4RN6Z+y+1YhCBoJHgfu42Lfi49swwK8A/U8GMCNKe +I7rnuRnp612ws1VKA+e9WStiQFMO7abvrxMdR+1CI6EVr443wwIREhUHKbSRtN66 +84BE4RdHeyPonGnikEJmsk7nrvQLIas0A9vahoPdJkHSAaSoI6dCsA3UIPCdhUxV +8EUFu1/bDwaILdbWB9dCGEozF6Z9V0MNZTy4lOkjI5vbOgrxwUqbi5WL7Ym+53Ff +F6XeortaeoUW5MOSt9qDZfvIiHtpbu3G9BYg0JJs1uhFmk+BCutm0rx85Q3u7r7H +HP7VkM2fFNXLOYZsF9yQHQ4hwyPAdCjj4/Q86QatV1d3y0yyDl8dWwkGkDLWtQjX +qpfMPEqlygAWJqClInQ8iTe66T8lTY5ZluAAB3kJrT2TjPJD33a7cU3Yi0l+jNgJ +RUHG0SkYVYLTVYN5zysO9w9XODdt2eKo7X/a1AF7TSaD7NYm0eD7tNiZg/NIZjwS +dLBgxZFDliMmB7vh+bSLwIVXDRtNkyG49HP/s69b3Y+xTWeJOvMiS3oEfGzxN8Ch +P3iuGqxkNDGCtXhjiUbC4ZVSULFOQ4yMxUXI3J2fA3S7dEGxbB8qOFesdRb8AQ4q +gbWKxQNOXf5pFzf9/Jwq3pGKoTH/7RCYrjmH6MIhiiYf1OENr8eJci8jyQWZvD/m +6Gzo3ajKCR7zIuMYj5d8l4numvY/g1xiKhNnJ1WE+oTDAViZ0ne0WC1ua5n09F2h +s7lNWxilFBhas2vbUqswrEKX9WlOO/BhiMweVRWC2/d0EQteg2msaFAc+aG+Db/u +mNvQOgOH763GpuA2NvlWRPyDSZJ789q+QsH0F6QZQDnWoVV6YkQM5Q/l/MRjvP9n +SoWGF5Cc8GePcsHrbZhrwzCoZ5VGahkpOB+HGxJ8OpqvVwqq7axSAk/PNQjiEwfG +93zqVuVC/bBecU42zWoeaDNbJftdngkLvKDAy/Mp5pucdGo//ssg1Hyp8PBmILq+ +8PHMpaApa593F2LjOFKE7IEKuEqyF07IfwTBY82OaYaEZ4FQZOXHps6jSjESSpHR ++ZFsOA6i51nJXhnodn9az5cLHOT1kBOA2+t6basuy9paUL5iDdBTLaukXsY/R1UB +btBEAurusZv4YOuDvjizVLd2iNvkHUfW5syILvsxZjrZDDI9ksGCY7l5iUp7d+9y +cuD/tlQiH3ZHaYgyeYUB54ecbhRQBLVgJdD7fOu69uGgM/A7VZ6sIJccCxeHeFBL +bVVi7Bpb0Q3hbIP1LL5ht7j2qa7iTMb3cHO1C07WcTkmeud/3bEo24WgIr75bby2 +K+ToTwgkIVCsyiw84hC9PBK+jIPXmVGnBsMX+4L26AtzK5+4GAqLMgmm1mqxFS8+ +FY7daSz656xzv9f5ksQZIry8PcLfqE8Cd7wlfEpW49mcaMARMklhnQARdSv0qf32 +pIvmnAsfhPxF+2SwOv5bkdeo4ixbC0RaYzl5Bj0xReRiF2M78eLnztNmstuoIRwX +DbMnOk0pXGyrPVdNkq9qacuxjisLEVx5Phb76VA71sXdNzosRdKRJTqY+AFl3CMZ +eSeTkBmGHP3nSviSJ2Hv3CMVHZw9hhHo1LW+lVtUX1fy7sAXq6wxaWqDrTakNAfn +1o+ICHJ1DeQWwSox7bdFhRnCrt4tb5Qta+BIK/2Ehhp2HmJZazAbMrB4FrwG2DQ8 +qyX1DKRhL9mpKmEcjoEU4ikFZHr8IuOs760dJKvI5lBD48ltJqrzxFTIPek+4yQI +YLFY1T1ZaE9E1O4wmnDGqwvjrYf2HsrAazG973/XnBgUC3jIDfbAWsS6Off7HFLb +czxzqs3vHuyzvgen4sDad5auLtVRiYdXasc4h/1Yl54x++MpRkXEwoIw2RGYNxQY +AXES2Jbtql2Lcjkwr3HQUJWMd0AtrHTfufvXBI+HsngYx+z9F+RamnAzph20squi +fb1ynvlUNc+YPbdgWI2UkQaNGSe6r0I+6zWi7RNuGK1Llipj1qcaEdBrUO6XQToh +9yQVflVU/GiXDwnMR/NljH7mBb2nZ6Rx25Na1JWWluluHr7d5FaJUQFz73T4/h7x +DvxbODqv9r4VepJ8uhCO2O3QECg9qUxeaN7brHbKKZTP38KYAPlFvBDJ3y1C/jPr +9ooeUKe7MR/PlyszOHvk+nqD0/QgLtsYajmOnbInk8+4kUNtpCpUMyzgouJgnmX+ +KJdUC0AU5ZnSnNZlJR7aaVNP1XwQbEZTcxyXJiwMlZNVVhQhSMr6lRSbDJGqDSIk +/4PGoKZXaToRxdy5jqakwJwP9L1Sa5DqL2ALZtaPEVK3NEFrjkRqgfiuEEezek2I +Y7U2dsN9lJBsK96//n5pKxz8ZRCogwD7tOVUYGc+f+YVaNa7UQ4I5cXKGfreS+TQ +JYwXhlZ8yUUu4dfpgPfiV0R9hzombUWwo9DeTvMYwGzDrW7rkCrz1BQDPhV4ItCm +rv1y+fjCRu4OuuWdYINrf3ejgix4O3ip7OQu+FszM9PACu7l2fOz6dcM9DZpuGLl +6weOuuxR99FiT58MVwcCU5siUxvQZz1WfTW1UatZZ58PRqnNcNAr4ideav9GvI82 +Ppo/KTHchkBr2e2RJVFeUjz+8ElN9KEK2TMIov2qXTA1L+Uw3AdssE+0ayenOk49 +JL/jc1eeY1bMPF45kPMniW+B3zhb54XQXcOCzMmMO7aap8dszIgxhVXvNkAp3gXg +Gx/ynKmrDgrjplddFHMQk8+7n8jqk2tvDSgjghHeh6GYjiwps0J0mh3Fs+mZY+AT +HnclmGq9NPE2UMA9eiQzakHxO8zpEGzV5RWifKYk6yCps4MLC1nVq76lyR8VoQr0 +Zj7G4OQIWI+miZEGUiuB5pW9Jj8BqXmJZW1EY0GAC712jLrpeMoaUiCP5pL4ZpzP +eJPBF1aBn3BxgfOU8dcTDLTQzl2VRspcvJEYdZwzgFAZOmTLgMn35CvmPsiwFyIr +t1CzQHoAYvY/VVskkCi0VtmL8Tk5mPn+w6QM3loIRlmMqd0nzKqtdtiF0V3w7Ily +L1OX/kG5Wz8QsyrAnT38BkDaHcv7Drvb+FSe+9j/+kzdbtlWY4ThFerbM31DO1qV +eud0p5U+kmPg8MBQ0BjW6TfqbC1Bk7J4XsUAa3RM5381+l+FObez33/1+S4Up/sM +Z3nEZmeeAbk+gAE01UyhbkgR9czLIjTTwv+zZRbBA0EkDC+Q+fBxX5nst6KC+jaF +kw2DhXQQpV83vqvBt+DWIqaQEjtsZICumtBBIosl3IRflQwqV85T1M0Bo8Tr6Rhh +xFgoFBkiO7CM9QuhvUb/xkV5gdMtnl8HgC35F8bdOdSnPdb03vXvGuzL3+0IzXcA +3elInCScL2RqKH5deD3SuoSqhqdN/7ZlbRICOdXLFlZARGvelCEIsPPPeQVIXoye +MVzXHCwMoWNJkGLCa2kTzf7Z5Frhr5baw3XAxKuqQhNGkliRDrFFnEWhpyJgHjlK +rWa4Ab3Ju5RgUIA66X3Zmsd3YGtF9J44LhI29Q2/wcC7A6Qx1eRRCXoGR2LT2hSM +V0ncIzBABPuQ+2RcwK/MZ1rFHBWQp4epMl3XPyVO5oVKduV0wnDOra0siALRXWb5 +ysMxVZYhXBtg+Dn2X6nKh/5rJqZg8/ZNM0NZICtpemSrVlHoY0/GYeES60ml7gE5 +ULQ/4Vyp1KjK8EosoXRNjiE7hQE8An9LqKuosVPJAMCBmX7bKl0XDsyTNk7zbPHO ++wHoR7Wc60DLMWv0YRIN5ZXcZEMel7dXzLyXPejrOdzS4W7KQFnOR6OXcpI7Mt2Z +HTdAWpLiwd3GkfvWx9a0aP0HqUYLjCJh7ApoVGtrUfPcAWU+cnuodQxLr3+agy4+ +52e3ItrmYt1vlPCcfBeJnRbEGI6ux3IKRT1EO7AG7CpyaAiau+4vGdCKU2rfLueq +S6+wI7Glp50BX7ljQtG4+wHtuaPrUc2Jsolm6W2rxCQHUxO30DbwZEXX118sXtLB +pzkCJHTAL42tTrUbIQ0Rj2FuZR0j0ZIDFYmOa7sLCvaY0xiqdJ/dI3UeFxr3Ehcf +zHfhpo7fTUqC7XpphA2BBb5gvxJcQY3dZlN8reRYq4g30/YPxjMrhoIwqxFLdvja +znSou7triyM9gOehBkRW6eg/3iyidWXCfzWAH1v+oy8mjrZvnYAtQZoFweIzUeUz +68GoM+QRZAiao6v0JK7Nwn24333+XFu6tIwf8yOsTvxQw/l1NJZQmIBg69WGo+gt +TNkcf1S3fmbVA079cxIp5SaPveCsER94Za7FIJG7qYry4VRyAyJnFBMiGttAw6OA +2RDDn9VpPGqxGawf8xHDia2OQOL2Sje3i+88pdAVtuzfZPQdimBzkszGwMT5wMvh +pIaRbtA8xK/r2emsgPMmjEbCYB0lBW/Q1j2rIfjwAWVRYmHFrKRoQV9C9K4nsHbs +921IsjdY0PIWmaYU0/ewiEP8xX1uao/lfWLciUGlpowGB9S/scqBphCZg5n5Qir6 +bSnK6FlpOuez9UH2SnJ+I44HdA7YMFvFqcsLfgkCKsVw3dQdAh2eB/jg0Gby4feN +rIts/8G0FDxLDhYwntMtUti3gFzoTkWokE2gv+JjnDQlYg6cYY0g/h2UhHM58Zgn +CqrC19KEy2E1mgsNhj8NJygsbk4Nr6oMxyCiKbHf8lProBMhhzj3SZB3hhRZSPBG +7SArHB/tepUwXh0NyyTioPdtejUL0iZTZAv8M2/Z+HKkHEfxwWJK8Tz/cwd6+EON +98jfawzErzbxkbBs5+I7vrfowy8cDGNV4sriPQFT1xKWo4d+hHfxnwc3eOQOZHdy +OVF/e9KlIrQi9ikIVo5TYoxH+l46Wr2J5t2OKLhkGZFQBf7dMCfij5T6bMoQh40V +ClVLkRttIYh+B7XsJZT+iN+lERoCdGNIDYRoqAfQ4bYkCBlBKbbHnl5lHDb1Sx8m +8ASUPKVoGGSf9IHGTdfzrMNKrtxMsRvAzcsRfaR6utKupv0n+qcSK+JYN6PqH6Rn +3aUnJWE9aoFlxK5i8DRseUIGB+XZeWP6RY61AGo= diff --git a/src/lib-dcrypt/test-crypto.c b/src/lib-dcrypt/test-crypto.c new file mode 100644 index 00000000000..501bae414b1 --- /dev/null +++ b/src/lib-dcrypt/test-crypto.c @@ -0,0 +1,560 @@ +#include "lib.h" +#include "buffer.h" +#include "str.h" +#include "dcrypt.h" +#include "ostream.h" +#include "ostream-encrypt.h" +#include "istream.h" +#include "istream-decrypt.h" +#include "iostream-temp.h" +#include "randgen.h" +#include "test-common.h" +#include "hex-binary.h" +#include +#include +#include + + +static +void test_cipher_test_vectors(void) +{ + static struct { + const char *key; + const char *iv; + const char *pt; + const char *ct; + } vectors[] = + { + { "2b7e151628aed2a6abf7158809cf4f3c", "000102030405060708090a0b0c0d0e0f", "6bc1bee22e409f96e93d7e117393172a", "7649abac8119b246cee98e9b12e9197d" }, + { "2b7e151628aed2a6abf7158809cf4f3c", "7649ABAC8119B246CEE98E9B12E9197D", "ae2d8a571e03ac9c9eb76fac45af8e51", "5086cb9b507219ee95db113a917678b2" } + }; + + + test_begin("test_cipher_test_vectors"); + + buffer_t *key,*iv,*pt,*ct,*res_enc,*res_dec; + + key = buffer_create_dynamic(pool_datastack_create(), 16); + iv = buffer_create_dynamic(pool_datastack_create(), 16); + pt = buffer_create_dynamic(pool_datastack_create(), 16); + ct = buffer_create_dynamic(pool_datastack_create(), 16); + + res_enc = buffer_create_dynamic(pool_datastack_create(), 32); + res_dec = buffer_create_dynamic(pool_datastack_create(), 32); + + for(size_t i = 0; i < N_ELEMENTS(vectors); i++) { + struct dcrypt_context_symmetric *ctx; + + buffer_set_used_size(key, 0); + buffer_set_used_size(iv, 0); + buffer_set_used_size(pt, 0); + buffer_set_used_size(ct, 0); + buffer_set_used_size(res_enc, 0); + buffer_set_used_size(res_dec, 0); + + hex_to_binary(vectors[i].key, key); + hex_to_binary(vectors[i].iv, iv); + hex_to_binary(vectors[i].pt, pt); + hex_to_binary(vectors[i].ct, ct); + + if (!dcrypt_ctx_sym_create("AES-128-CBC", DCRYPT_MODE_ENCRYPT, &ctx, NULL)) { + test_assert_failed("dcrypt_ctx_sym_create", __FILE__, __LINE__-1); + continue; + } + + dcrypt_ctx_sym_set_padding(ctx, FALSE); + + dcrypt_ctx_sym_set_key(ctx, key->data, key->used); + dcrypt_ctx_sym_set_iv(ctx, iv->data, iv->used); + + test_assert_idx(dcrypt_ctx_sym_init(ctx, NULL), i); + + test_assert_idx(dcrypt_ctx_sym_update(ctx, pt->data, pt->used, res_enc, NULL), i); + test_assert_idx(dcrypt_ctx_sym_final(ctx, res_enc, NULL), i); + + test_assert_idx(buffer_cmp(ct, res_enc), i); + + dcrypt_ctx_sym_destroy(&ctx); + + if (!dcrypt_ctx_sym_create("AES-128-CBC", DCRYPT_MODE_DECRYPT, &ctx, NULL)) { + test_assert_failed("dcrypt_ctx_sym_create", __FILE__, __LINE__-1); + continue; + } + + dcrypt_ctx_sym_set_padding(ctx, FALSE); + + dcrypt_ctx_sym_set_key(ctx, key->data, key->used); + dcrypt_ctx_sym_set_iv(ctx, iv->data, iv->used); + + test_assert_idx(dcrypt_ctx_sym_init(ctx, NULL), i); + test_assert_idx(dcrypt_ctx_sym_update(ctx, res_enc->data, res_enc->used, res_dec, NULL), i); + test_assert_idx(dcrypt_ctx_sym_final(ctx, res_dec, NULL), i); + + test_assert_idx(buffer_cmp(pt, res_dec), i); + + dcrypt_ctx_sym_destroy(&ctx); + } + + test_end(); +} + +static +void test_cipher_aead_test_vectors(void) +{ + struct dcrypt_context_symmetric *ctx; + const char *error = NULL; + + test_begin("test_cipher_aead_test_vectors"); + + if (!dcrypt_ctx_sym_create("aes-128-gcm", DCRYPT_MODE_ENCRYPT, &ctx, &error)) { + test_assert_failed("dcrypt_ctx_sym_create", __FILE__, __LINE__-1); + return; + } + + buffer_t *key, *iv, *aad, *pt, *ct, *tag, *tag_res, *res; + + key = buffer_create_dynamic(pool_datastack_create(), 16); + iv = buffer_create_dynamic(pool_datastack_create(), 16); + aad = buffer_create_dynamic(pool_datastack_create(), 16); + pt = buffer_create_dynamic(pool_datastack_create(), 16); + ct = buffer_create_dynamic(pool_datastack_create(), 16); + tag = buffer_create_dynamic(pool_datastack_create(), 16); + res = buffer_create_dynamic(pool_datastack_create(), 16); + tag_res = buffer_create_dynamic(pool_datastack_create(), 16); + + hex_to_binary("feffe9928665731c6d6a8f9467308308", key); + hex_to_binary("cafebabefacedbaddecaf888", iv); + hex_to_binary("d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b391aafd255", pt); + hex_to_binary("42831ec2217774244b7221b784d0d49ce3aa212f2c02a4e035c17e2329aca12e21d514b25466931c7d8f6a5aac84aa051ba30b396a0aac973d58e091473f5985", ct); + hex_to_binary("4d5c2af327cd64a62cf35abd2ba6fab4", tag); + + dcrypt_ctx_sym_set_key(ctx, key->data, key->used); + dcrypt_ctx_sym_set_iv(ctx, iv->data, iv->used); + dcrypt_ctx_sym_set_aad(ctx, aad->data, aad->used); + test_assert(dcrypt_ctx_sym_init(ctx, &error)); + test_assert(dcrypt_ctx_sym_update(ctx, pt->data, pt->used, res, &error)); + test_assert(dcrypt_ctx_sym_final(ctx, res, &error)); + test_assert(dcrypt_ctx_sym_get_tag(ctx, tag_res)); + + test_assert(buffer_cmp(ct, res) == TRUE); + test_assert(buffer_cmp(tag, tag_res) == TRUE); + + dcrypt_ctx_sym_destroy(&ctx); + + if (!dcrypt_ctx_sym_create("aes-128-gcm", DCRYPT_MODE_DECRYPT, &ctx, &error)) { + test_assert_failed("dcrypt_ctx_sym_create", __FILE__, __LINE__-1); + } else { + + buffer_set_used_size(res, 0); + + dcrypt_ctx_sym_set_key(ctx, key->data, key->used); + dcrypt_ctx_sym_set_iv(ctx, iv->data, iv->used); + dcrypt_ctx_sym_set_aad(ctx, aad->data, aad->used); + dcrypt_ctx_sym_set_tag(ctx, tag->data, tag->used); + test_assert(dcrypt_ctx_sym_init(ctx, &error)); + test_assert(dcrypt_ctx_sym_update(ctx, ct->data, ct->used, res, &error)); + test_assert(dcrypt_ctx_sym_final(ctx, res, &error)); + + test_assert(buffer_cmp(pt, res) == TRUE); + + dcrypt_ctx_sym_destroy(&ctx); + } + + test_end(); +} + +static +void test_hmac_test_vectors(void) +{ + test_begin("test_hmac_test_vectors"); + + buffer_t *pt, *ct, *key, *res; + pt = buffer_create_dynamic(pool_datastack_create(), 50); + key = buffer_create_dynamic(pool_datastack_create(), 20); + ct = buffer_create_dynamic(pool_datastack_create(), 32); + res = buffer_create_dynamic(pool_datastack_create(), 32); + + hex_to_binary("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", key); + hex_to_binary("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", pt); + hex_to_binary("773ea91e36800e46854db8ebd09181a72959098b3ef8c122d9635514ced565fe", res); + + struct dcrypt_context_hmac *hctx; + if (!dcrypt_ctx_hmac_create("sha256", &hctx, NULL)) { + test_assert_failed("dcrypt_ctx_hmac_create", __FILE__, __LINE__-1); + } else { + dcrypt_ctx_hmac_set_key(hctx, key->data, key->used); + test_assert(dcrypt_ctx_hmac_init(hctx, NULL)); + test_assert(dcrypt_ctx_hmac_update(hctx, pt->data, pt->used, NULL)); + test_assert(dcrypt_ctx_hmac_final(hctx, ct, NULL)); + test_assert(buffer_cmp(ct, res)); + dcrypt_ctx_hmac_destroy(&hctx); + } + + test_end(); +} + +static +void test_load_v1_keys(void) +{ + test_begin("test_load_v1_keys"); + + const char *error = NULL; + const char *data1 = "1\t716\t1\t0567e6bf9579813ae967314423b0fceb14bda24749303923de9a9bb9370e0026f995901a57e63113eeb2baf0c940e978d00686cbb52bd5014bc318563375876255\t0300E46DA2125427BE968EB3B649910CDC4C405E5FFDE18D433A97CABFEE28CEEFAE9EE356C792004FFB80981D67E741B8CC036A34235A8D2E1F98D1658CFC963D07EB\td0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0\t7c9a1039ea2e4fed73e81dd3ffc3fa22ea4a28352939adde7bf8ea858b00fa4f"; + + enum dcrypt_key_format format; + enum dcrypt_key_version version; + enum dcrypt_key_kind kind; + enum dcrypt_key_encryption_type encryption_type; + const char *encryption_key_hash = NULL; + const char *key_hash = NULL; + + bool ret = dcrypt_key_string_get_info(data1, &format, &version, + &kind, &encryption_type, &encryption_key_hash, + &key_hash, &error); + + test_assert(ret == TRUE); + test_assert(error == NULL); + test_assert(format == DCRYPT_FORMAT_DOVECOT); + test_assert(version == DCRYPT_KEY_VERSION_1); + test_assert(kind == DCRYPT_KEY_KIND_PRIVATE); + test_assert(encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_KEY); + test_assert(strcmp(encryption_key_hash, "d0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0") == 0); + test_assert(strcmp(key_hash, "7c9a1039ea2e4fed73e81dd3ffc3fa22ea4a28352939adde7bf8ea858b00fa4f") == 0); + + const char* data2 = "1\t716\t0301EB00973C4EFC8FCECA4EA33E941F50B561199A5159BCB6C2EED9DD1D62D65E38A254979D89E28F0C28883E71EE2AD264CD16B863FA094A8F6F69A56B62E8918040\t7c9a1039ea2e4fed73e81dd3ffc3fa22ea4a28352939adde7bf8ea858b00fa4f"; + + error = NULL; + encryption_key_hash = NULL; + key_hash = NULL; + + ret = dcrypt_key_string_get_info(data2, &format, &version, + &kind, &encryption_type, &encryption_key_hash, + &key_hash, &error); + + test_assert(ret == TRUE); + test_assert(error == NULL); + test_assert(format == DCRYPT_FORMAT_DOVECOT); + test_assert(version == DCRYPT_KEY_VERSION_1); + test_assert(kind == DCRYPT_KEY_KIND_PUBLIC); + test_assert(encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_NONE); + test_assert(encryption_key_hash == NULL); + test_assert(strcmp(key_hash, "7c9a1039ea2e4fed73e81dd3ffc3fa22ea4a28352939adde7bf8ea858b00fa4f") == 0); + + /* This is the key that should be able to decrypt key1 */ + const char *data3 = "1\t716\t0\t048FD04FD3612B22D32790C592CF21CEF417EFD2EA34AE5F688FA5B51BED29E05A308B68DA78E16E90B47A11E133BD9A208A2894FD01B0BEE865CE339EA3FB17AC\td0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0"; + + error = NULL; + encryption_key_hash = NULL; + key_hash = NULL; + + ret = dcrypt_key_string_get_info(data3, &format, &version, + &kind, &encryption_type, &encryption_key_hash, + &key_hash, &error); + test_assert(ret == TRUE); + test_assert(error == NULL); + test_assert(format == DCRYPT_FORMAT_DOVECOT); + test_assert(version == DCRYPT_KEY_VERSION_1); + test_assert(kind == DCRYPT_KEY_KIND_PRIVATE); + test_assert(encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_NONE); + test_assert(encryption_key_hash == NULL); + test_assert(strcmp(key_hash, "d0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0") == 0); + + /* key3's key_hash should and does match key1's encryption_key_hash */ + struct dcrypt_private_key *pkey = NULL; + struct dcrypt_private_key *pkey2 = NULL; + pkey = NULL; + error = NULL; + + ret = dcrypt_key_load_private(&pkey2, format, data3, NULL, NULL, &error); + test_assert(ret == TRUE); + test_assert(error == NULL); + + ret = dcrypt_key_load_private(&pkey, format, data1, NULL, pkey2, &error); + test_assert(ret == TRUE); + test_assert(error == NULL); + + dcrypt_key_free_private(&pkey2); + dcrypt_key_free_private(&pkey); + + test_end(); +} + +static +void test_load_v1_key(void) +{ + test_begin("test_load_v1_key"); + + buffer_t *key_1 = buffer_create_dynamic(pool_datastack_create(), 128); + + struct dcrypt_private_key *pkey = NULL, *pkey2 = NULL; + const char *error = NULL; + + test_assert(dcrypt_key_load_private(&pkey, DCRYPT_FORMAT_DOVECOT, "1\t716\t0\t048FD04FD3612B22D32790C592CF21CEF417EFD2EA34AE5F688FA5B51BED29E05A308B68DA78E16E90B47A11E133BD9A208A2894FD01B0BEE865CE339EA3FB17AC\td0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0", NULL, NULL, &error)); + if (pkey != NULL) { + buffer_set_used_size(key_1, 0); + /* check that key_id matches */ + struct dcrypt_public_key *pubkey = NULL; + dcrypt_key_convert_private_to_public(pkey, &pubkey); + test_assert(dcrypt_key_store_public(pubkey, DCRYPT_FORMAT_DOVECOT, key_1, NULL)); + buffer_set_used_size(key_1, 0); + dcrypt_key_id_public(pubkey, "sha256", key_1, &error); + test_assert(strcmp("792caad4d38c9eb2134a0cbc844eae386116de096a0ccafc98479825fc99b6a1", binary_to_hex(key_1->data, key_1->used)) == 0); + + dcrypt_key_free_public(&pubkey); + pkey2 = NULL; + + test_assert(dcrypt_key_load_private(&pkey2, DCRYPT_FORMAT_DOVECOT, "1\t716\t1\t0567e6bf9579813ae967314423b0fceb14bda24749303923de9a9bb9370e0026f995901a57e63113eeb2baf0c940e978d00686cbb52bd5014bc318563375876255\t0300E46DA2125427BE968EB3B649910CDC4C405E5FFDE18D433A97CABFEE28CEEFAE9EE356C792004FFB80981D67E741B8CC036A34235A8D2E1F98D1658CFC963D07EB\td0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0\t7c9a1039ea2e4fed73e81dd3ffc3fa22ea4a28352939adde7bf8ea858b00fa4f", NULL, pkey, &error)); + if (pkey2 != NULL) { + buffer_set_used_size(key_1, 0); + /* check that key_id matches */ + struct dcrypt_public_key *pubkey = NULL; + dcrypt_key_convert_private_to_public(pkey2, &pubkey); + test_assert(dcrypt_key_store_public(pubkey, DCRYPT_FORMAT_DOVECOT, key_1, NULL)); + buffer_set_used_size(key_1, 0); + test_assert(dcrypt_key_id_public_old(pubkey, key_1, &error)); + test_assert(strcmp("7c9a1039ea2e4fed73e81dd3ffc3fa22ea4a28352939adde7bf8ea858b00fa4f", binary_to_hex(key_1->data, key_1->used)) == 0); + + dcrypt_key_free_public(&pubkey); + dcrypt_key_free_private(&pkey2); + } + dcrypt_key_free_private(&pkey); + } + + test_end(); +} + +static +void test_load_v1_public_key(void) +{ + test_begin("test_load_v1_public_key"); + + const char* data1 = "1\t716\t030131D8A5FD5167947A0AE9CB112ADED6526654635AA5887051EE2364414B60FF32EBA8FA0BBE9485DBDE8794BBBCB44BBFC0D662A4287A848BA570D4E5E45A11FE0F\td0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0"; + + const char* error = NULL; + const char* key_hash = NULL; + const char* encryption_key_hash = NULL; + + enum dcrypt_key_format format; + enum dcrypt_key_version version; + enum dcrypt_key_kind kind; + enum dcrypt_key_encryption_type encryption_type; + + bool ret = dcrypt_key_string_get_info(data1, &format, &version, + &kind, &encryption_type, &encryption_key_hash, + &key_hash, &error); + + test_assert(ret == TRUE); + test_assert(error == NULL); + test_assert(format == DCRYPT_FORMAT_DOVECOT); + test_assert(version == DCRYPT_KEY_VERSION_1); + test_assert(kind == DCRYPT_KEY_KIND_PUBLIC); + test_assert(encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_NONE); + test_assert(key_hash != NULL && strcmp(key_hash, "d0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0") == 0); + test_assert(encryption_key_hash == NULL); + + struct dcrypt_public_key *pub_key = NULL; + ret = dcrypt_key_load_public(&pub_key, format, data1, &error); + test_assert(ret == TRUE); + test_assert(error == NULL); + + test_assert(dcrypt_key_type_public(pub_key) == DCRYPT_KEY_EC); + + dcrypt_key_free_public(&pub_key); + test_assert(pub_key == NULL); + + test_end(); +} + +static +void test_load_v2_key(void) +{ + const char *keys[] = { + "-----BEGIN PRIVATE KEY-----\n" \ +"MGcCAQAwEwYHKoZIzj0CAQYIKoZIzj0DAQcETTBLAgEBBCC25AkD65uhlZXCAdwN\n" \ +"yLJV2ui8A/CUyqyEMrezvwgMO6EkAyIAAybRUR3MsH0+0PQcDwkrXOJ9aePwzTQV\n" \ +"DN51+n1JCxbI\n" \ +"-----END PRIVATE KEY-----\n", + "2\t1.2.840.10045.3.1.7\t0\t0000002100b6e40903eb9ba19595c201dc0dc8b255dae8bc03f094caac8432b7b3bf080c3b\tab13d251976dedab546b67354e7678821740dd534b749c2857f66bf62bbaddfd", + "2\t1.2.840.10045.3.1.7\t2\taes-256-ctr\t2b19763d4bbf7754\tsha256\t2048\tc36fa194669a1aec400eae32fbadaa7c58b14f53c464cfbb0a4b61fbe24ab7750637c4025d\tab13d251976dedab546b67354e7678821740dd534b749c2857f66bf62bbaddfd", + "2\t1.2.840.10045.3.1.7\t1\taes-256-ctr\t7c7f1d12a7c011de\tsha256\t2048\tf5d1de11d58a81b141cf038012a618623e9d7b18062deeb3a4e35872c62ca0837db8688370\t021abfbc5bc4f6cf49c40b9fc388c4616ea079941675f477ee4557df1919626d35\tab13d251976dedab546b67354e7678821740dd534b749c2857f66bf62bbaddfd\tab13d251976dedab546b67354e7678821740dd534b749c2857f66bf62bbaddfd" + }; + + test_begin("test_load_v2_key"); + const char *error = NULL; + buffer_t *tmp = buffer_create_dynamic(default_pool, 256); + + struct dcrypt_private_key *priv,*priv2; + + test_assert_idx(dcrypt_key_load_private(&priv2, DCRYPT_FORMAT_PEM, keys[0], NULL, NULL, &error), 0); + test_assert_idx(dcrypt_key_store_private(priv2, DCRYPT_FORMAT_PEM, NULL, tmp, NULL, NULL, &error), 0); + test_assert_idx(strcmp(str_c(tmp), keys[0])==0, 0); + buffer_set_used_size(tmp, 0); + + test_assert_idx(dcrypt_key_load_private(&priv, DCRYPT_FORMAT_DOVECOT, keys[1], NULL, NULL, &error), 1); + test_assert_idx(dcrypt_key_store_private(priv, DCRYPT_FORMAT_DOVECOT, NULL, tmp, NULL, NULL, &error), 1); + test_assert_idx(strcmp(str_c(tmp), keys[1])==0, 1); + buffer_set_used_size(tmp, 0); + dcrypt_key_free_private(&priv); + + test_assert_idx(dcrypt_key_load_private(&priv, DCRYPT_FORMAT_DOVECOT, keys[2], "This Is Sparta", NULL, &error), 2); + test_assert_idx(dcrypt_key_store_private(priv, DCRYPT_FORMAT_DOVECOT, "aes-256-ctr", tmp, "This Is Sparta", NULL, &error), 2); + buffer_set_used_size(tmp, 0); + dcrypt_key_free_private(&priv); + + struct dcrypt_public_key *pub = NULL; + dcrypt_key_convert_private_to_public(priv2, &pub); + test_assert_idx(dcrypt_key_load_private(&priv, DCRYPT_FORMAT_DOVECOT, keys[3], NULL, priv2, &error), 3); + test_assert_idx(dcrypt_key_store_private(priv, DCRYPT_FORMAT_DOVECOT, "ecdh-aes-256-ctr", tmp, NULL, pub, &error), 3); + buffer_set_used_size(tmp, 0); + dcrypt_key_free_private(&priv2); + dcrypt_key_free_private(&priv); + dcrypt_key_free_public(&pub); + + buffer_free(&tmp); + + if (error != NULL) error = NULL; + + test_end(); +} + +static +void test_load_v2_public_key(void) +{ + struct dcrypt_public_key *pub = NULL; + const char *error; + + test_begin("test_load_v2_public_key"); + const char *key = "2\t3058301006072a8648ce3d020106052b810400230344000301c50954e734dd8b410a607764a7057065a45510da52f2c6e28e0cb353b9c389fa8cb786943ae991fce9befed78fb162fbbc615415f06af06c8cc80c37f4e94ff6c7\t185a7212542782e239111f9c19d126ad55b18ddaf4883d66afe8d9627c3607d8"; + + test_assert(dcrypt_key_load_public(&pub, DCRYPT_FORMAT_DOVECOT, key, &error)); + + buffer_t *tmp = buffer_create_dynamic(default_pool, 256); + + if (pub != NULL) { + test_assert(dcrypt_key_store_public(pub, DCRYPT_FORMAT_DOVECOT, tmp, &error)); + test_assert(strcmp(key, str_c(tmp))==0); + buffer_free(&tmp); + dcrypt_key_free_public(&pub); + } + + test_end(); +} + +static +void test_get_info_v2_key(void) { + test_begin("test_get_info_v2_key"); + + const char *key = "2\t305e301006072a8648ce3d020106052b81040026034a000203fcc90034fa03d6fb79a0fc8b3b43c3398f68e76029307360cdcb9e27bb7e84b3c19dfb7244763bc4d442d216f09b7b7945ed9d182f3156550e9ee30b237a0217dbf79d28975f31\t86706b69d1f640011a65d26a42f2ba20a619173644e1cc7475eb1d90966e84dc"; + enum dcrypt_key_format format; + enum dcrypt_key_version version = DCRYPT_KEY_VERSION_NA; + enum dcrypt_key_kind kind; + enum dcrypt_key_encryption_type encryption_type; + const char *encryption_key_hash = NULL; + const char *key_hash = NULL; + const char *error = NULL; + + test_assert(dcrypt_key_string_get_info(key, &format, &version, + &kind, &encryption_type, &encryption_key_hash, + &key_hash, &error)); + test_assert(error == NULL); + test_assert(format == DCRYPT_FORMAT_DOVECOT); + test_assert(version == DCRYPT_KEY_VERSION_2); + + test_assert(kind == DCRYPT_KEY_KIND_PUBLIC); + test_assert(encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_NONE); + test_assert(encryption_key_hash == NULL); + test_assert(key_hash != NULL && strcmp(key_hash, + "86706b69d1f640011a65d26a42f2ba20a619173644e1cc7475eb1d90966e84dc") == 0); + + test_end(); +} + +static +void test_gen_and_get_info_rsa_pem(void) +{ + test_begin("test_gen_and_get_info_rsa_pem"); + + const char *error = NULL; + bool ret = FALSE; + struct dcrypt_keypair pair; + string_t* buf = str_new(default_pool, 4096); + + ret = dcrypt_keypair_generate(&pair, DCRYPT_KEY_RSA, 1024, NULL, NULL); + test_assert(ret == TRUE); + + /* test public key */ + enum dcrypt_key_format format; + enum dcrypt_key_version version; + enum dcrypt_key_kind kind; + enum dcrypt_key_encryption_type encryption_type; + const char *encryption_key_hash; + const char *key_hash; + + ret = dcrypt_key_store_public(pair.pub, DCRYPT_FORMAT_PEM, buf, + &error); + test_assert(ret == TRUE); + + ret = dcrypt_key_string_get_info(str_c(buf), &format, &version, + &kind, &encryption_type, &encryption_key_hash, + &key_hash, &error); + test_assert(ret == TRUE); + test_assert(format == DCRYPT_FORMAT_PEM); + test_assert(version == DCRYPT_KEY_VERSION_NA); + + test_assert(kind == DCRYPT_KEY_KIND_PUBLIC); + test_assert(encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_NONE); + test_assert(encryption_key_hash == NULL); + test_assert(key_hash == NULL); + + /* test private key */ + buffer_set_used_size(buf, 0); + ret = dcrypt_key_store_private(pair.priv, DCRYPT_FORMAT_PEM, NULL, + buf, NULL, NULL, &error); + + test_assert(ret == TRUE); + + ret = dcrypt_key_string_get_info(str_c(buf), &format, &version, + &kind, &encryption_type, &encryption_key_hash, + &key_hash, &error); + + test_assert(ret == TRUE); + test_assert(format == DCRYPT_FORMAT_PEM); + test_assert(version == DCRYPT_KEY_VERSION_NA); + + test_assert(kind == DCRYPT_KEY_KIND_PRIVATE); + + test_assert(encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_NONE); + test_assert(encryption_key_hash == NULL); + test_assert(key_hash == NULL); + + dcrypt_keypair_free(&pair); + buffer_free(&buf); + + test_end(); +} + +int main(void) { + random_init(); + dcrypt_initialize("openssl", NULL, NULL); + + static void (*test_functions[])(void) = { + test_cipher_test_vectors, + test_cipher_aead_test_vectors, + test_hmac_test_vectors, + test_load_v1_keys, + test_load_v1_key, + test_load_v1_public_key, + test_load_v2_key, + test_load_v2_public_key, + test_get_info_v2_key, + test_gen_and_get_info_rsa_pem, + NULL + }; + + int ret = test_run(test_functions); + + dcrypt_deinitialize(); + random_deinit(); + + return ret; +} diff --git a/src/lib-dcrypt/test-stream.c b/src/lib-dcrypt/test-stream.c new file mode 100644 index 00000000000..e078a8f21a0 --- /dev/null +++ b/src/lib-dcrypt/test-stream.c @@ -0,0 +1,427 @@ +#include "lib.h" +#include "buffer.h" +#include "str.h" +#include "dcrypt.h" +#include "ostream.h" +#include "ostream-encrypt.h" +#include "istream.h" +#include "istream-decrypt.h" +#include "istream-hash.h" +#include "istream-base64.h" +#include "iostream-temp.h" +#include "randgen.h" +#include "hash-method.h" +#include "test-common.h" +#include "hex-binary.h" +#include +#include +#include + +static const char key_v1_priv[] = "-----BEGIN PRIVATE KEY-----\n" \ +"MIGpAgEAMBAGByqGSM49AgEGBSuBBAAjBIGRMIGOAgEBBEGz2V2VMi/5s+Z+GJh7\n" \ +"4WfqZjZUpqqm+NJWojm6BbrZMY+9ZComlTGVcUZ007acFxV93oMmrfmtRUb5ynrb\n" \ +"MRFskKFGA0QAAwHrAJc8TvyPzspOoz6UH1C1YRmaUVm8tsLu2d0dYtZeOKJUl52J\n" \ +"4o8MKIg+ce4q0mTNFrhj+glKj29ppWti6JGAQA==\n" \ +"-----END PRIVATE KEY-----"; + +static const char key_v1_pub[] = "-----BEGIN PUBLIC KEY-----\n" \ +"MFgwEAYHKoZIzj0CAQYFK4EEACMDRAADAesAlzxO/I/Oyk6jPpQfULVhGZpRWby2\n" \ +"wu7Z3R1i1l44olSXnYnijwwoiD5x7irSZM0WuGP6CUqPb2mla2LokYBA\n" \ +"-----END PUBLIC KEY-----"; + +static const char key_v2_priv[] = "-----BEGIN PRIVATE KEY-----\n" \ +"MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgtuQJA+uboZWVwgHc\n" \ +"DciyVdrovAPwlMqshDK3s78IDDuhRANCAAQm0VEdzLB9PtD0HA8JK1zifWnj8M00\n" \ +"FQzedfp9SQsWyA8dzs5/NFR5MTe6Xbh/ndKEs1zZH3vZ4FlNrilZc0st\n" \ +"-----END PRIVATE KEY-----"; + +static const char key_v2_pub[] = "-----BEGIN PUBLIC KEY-----\n" \ +"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJtFRHcywfT7Q9BwPCStc4n1p4/DN\n" \ +"NBUM3nX6fUkLFsgPHc7OfzRUeTE3ul24f53ShLNc2R972eBZTa4pWXNLLQ==\n" \ +"-----END PUBLIC KEY-----"; + +static const char test_sample_v1_hash[] = "1d7cc2cc1f1983f76241cc42389911e88590ad58cf9d54cafeb5b198d3723dd1"; +static const char test_sample_v1_short_hash[] = "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"; +static const char test_sample_v2_hash[] = "2e31218656dd34db65b321688bf418dee4ee785e99eb9c21e0d29b4af27a863e"; + +static struct dcrypt_keypair test_v1_kp; +static struct dcrypt_keypair test_v2_kp; + +static +void test_static_v1_input(void) +{ + ssize_t siz; + const struct hash_method *hash = hash_method_lookup("sha256"); + unsigned char hash_ctx[hash->context_size]; + unsigned char hash_dgst[hash->digest_size]; + hash->init(hash_ctx); + + test_begin("test_static_v1_input"); + + struct istream *is_1 = i_stream_create_file(DCRYPT_SRC_DIR"/sample-v1.asc", IO_BLOCK_SIZE); + struct istream *is_2 = i_stream_create_base64_decoder(is_1); + i_stream_unref(&is_1); + struct istream *is_3 = i_stream_create_decrypt(is_2, test_v1_kp.priv); + i_stream_unref(&is_2); + struct istream *is_4 = i_stream_create_hash(is_3, hash, hash_ctx); + i_stream_unref(&is_3); + + while((siz = i_stream_read(is_4))>0) { i_stream_skip(is_4, siz); } + + if (is_4->stream_errno != 0) + i_debug("error: %s", i_stream_get_error(is_4)); + + test_assert(is_4->stream_errno == 0); + + i_stream_unref(&is_4); + + hash->result(hash_ctx, hash_dgst); + + test_assert(strcmp(test_sample_v1_hash, binary_to_hex(hash_dgst, sizeof(hash_dgst))) == 0); + + test_end(); +} + +static +void test_static_v1_input_short(void) +{ + ssize_t siz; + const struct hash_method *hash = hash_method_lookup("sha256"); + unsigned char hash_ctx[hash->context_size]; + unsigned char hash_dgst[hash->digest_size]; + hash->init(hash_ctx); + + test_begin("test_static_v1_input_short"); + + struct istream *is_1 = i_stream_create_file(DCRYPT_SRC_DIR"/sample-v1_short.asc", IO_BLOCK_SIZE); + struct istream *is_2 = i_stream_create_base64_decoder(is_1); + i_stream_unref(&is_1); + struct istream *is_3 = i_stream_create_decrypt(is_2, test_v1_kp.priv); + i_stream_unref(&is_2); + struct istream *is_4 = i_stream_create_hash(is_3, hash, hash_ctx); + i_stream_unref(&is_3); + + while((siz = i_stream_read(is_4))>0) { i_stream_skip(is_4, siz); } + + if (is_4->stream_errno != 0) + i_debug("error: %s", i_stream_get_error(is_4)); + + test_assert(is_4->stream_errno == 0); + + i_stream_unref(&is_4); + + hash->result(hash_ctx, hash_dgst); + + test_assert(strcmp(test_sample_v1_short_hash, binary_to_hex(hash_dgst, sizeof(hash_dgst))) == 0); + + test_end(); +} + +static +void test_static_v2_input(void) +{ + test_begin("test_static_v2_input"); + ssize_t amt; + const struct hash_method *hash = hash_method_lookup("sha256"); + unsigned char hash_ctx[hash->context_size]; + unsigned char hash_dgst[hash->digest_size]; + hash->init(hash_ctx); + + struct istream *is_1 = i_stream_create_file(DCRYPT_SRC_DIR"/sample-v2.asc", IO_BLOCK_SIZE); + struct istream *is_2 = i_stream_create_base64_decoder(is_1); + i_stream_unref(&is_1); + struct istream *is_3 = i_stream_create_decrypt(is_2, test_v2_kp.priv); + i_stream_unref(&is_2); + struct istream *is_4 = i_stream_create_hash(is_3, hash, hash_ctx); + i_stream_unref(&is_3); + + while((amt = i_stream_read(is_4))>0) { i_stream_skip(is_4, amt); } + + if (is_4->stream_errno != 0) + i_debug("error: %s", i_stream_get_error(is_4)); + + test_assert(is_4->stream_errno == 0); + + i_stream_unref(&is_4); + + hash->result(hash_ctx, hash_dgst); + + test_assert(strcmp(test_sample_v2_hash, binary_to_hex(hash_dgst, sizeof(hash_dgst))) == 0); + + test_end(); + +/** this code is left here to show how the sample file is created + struct istream *is = i_stream_create_file("../lib-fts/udhr_fra.txt", 8192); + struct istream *is_2 = i_stream_create_hash(is, hash, hash_ctx); + int fd = open("sample-v2.bin", O_CREAT|O_TRUNC|O_WRONLY, S_IRWXU); + struct ostream *os = o_stream_create_fd_file(fd, 0, TRUE); + struct ostream *os_2 = o_stream_create_encrypt(os, "aes-256-gcm-sha256", test_v2_kp.pub, IO_STREAM_ENC_INTEGRITY_AEAD); + const unsigned char *ptr; + size_t siz; + + while(i_stream_read_data(is_2, &ptr, &siz, 0)>0) { + o_stream_nsend(os_2, ptr, siz); + i_stream_skip(is_2, siz); + } + + i_assert(o_stream_nfinish(os_2)==0); + + o_stream_close(os_2); + i_stream_close(is_2); + + hash->result(hash_ctx, hash_dgst); + printf("%s\n", binary_to_hex(hash_dgst, sizeof(hash_dgst))); +*/ +} + +static +void test_write_read_v1(void) +{ + test_begin("test_write_read_v1"); + unsigned char payload[IO_BLOCK_SIZE]; + const unsigned char *ptr; + size_t pos = 0, siz; + random_fill_weak(payload, IO_BLOCK_SIZE); + + struct ostream *os = iostream_temp_create("/tmp", 0); + struct ostream *os_2 = o_stream_create_encrypt(os, "", test_v2_kp.pub, IO_STREAM_ENC_VERSION_1); + o_stream_nsend(os_2, payload, sizeof(payload)); + + if (os_2->stream_errno != 0) + i_debug("error: %s", o_stream_get_error(os_2)); + + test_assert(os_2->stream_errno == 0); + test_assert(o_stream_nfinish(os_2) == 0); + test_assert(os_2->stream_errno == 0); + + o_stream_unref(&os_2); + + struct istream *is = iostream_temp_finish(&os, IO_BLOCK_SIZE); + struct istream *is_2 = i_stream_create_decrypt(is, test_v2_kp.priv); + i_stream_unref(&is); + + while(i_stream_read_data(is_2, &ptr, &siz, 0)>0) { + test_assert_idx(pos + siz <= sizeof(payload), pos); + if (pos + siz > sizeof(payload)) break; + test_assert_idx(memcmp(ptr, payload + pos, siz) == 0, pos); + i_stream_skip(is_2, siz); + } + + test_assert(is_2->stream_errno == 0); + + i_stream_unref(&is_2); + + test_end(); +} + +static +void test_write_read_v1_short(void) +{ + test_begin("test_write_read_v1_short"); + unsigned char payload[1]; + const unsigned char *ptr; + size_t pos = 0, siz; + random_fill_weak(payload, 1); + + struct ostream *os = iostream_temp_create("/tmp", 0); + struct ostream *os_2 = o_stream_create_encrypt(os, "", test_v2_kp.pub, IO_STREAM_ENC_VERSION_1); + o_stream_nsend(os_2, payload, sizeof(payload)); + + if (os_2->stream_errno != 0) + i_debug("error: %s", o_stream_get_error(os_2)); + + test_assert(os_2->stream_errno == 0); + test_assert(o_stream_nfinish(os_2) == 0); + test_assert(os_2->stream_errno == 0); + + o_stream_unref(&os_2); + + struct istream *is = iostream_temp_finish(&os, IO_BLOCK_SIZE); + struct istream *is_2 = i_stream_create_decrypt(is, test_v2_kp.priv); + i_stream_unref(&is); + + while(i_stream_read_data(is_2, &ptr, &siz, 0)>0) { + test_assert_idx(pos + siz <= sizeof(payload), pos); + if (pos + siz > sizeof(payload)) break; + test_assert_idx(memcmp(ptr, payload + pos, siz) == 0, pos); + i_stream_skip(is_2, siz); + } + + test_assert(is_2->stream_errno == 0); + + i_stream_unref(&is_2); + + test_end(); +} + +static +void test_write_read_v1_empty(void) +{ + const unsigned char *ptr; + size_t siz; + test_begin("test_write_read_v1_empty"); + struct ostream *os = iostream_temp_create("/tmp", 0); + struct ostream *os_2 = o_stream_create_encrypt(os, "", test_v1_kp.pub, IO_STREAM_ENC_VERSION_1); + test_assert(o_stream_nfinish(os_2) == 0); + if (os_2->stream_errno != 0) + i_debug("error: %s", o_stream_get_error(os_2)); + + o_stream_unref(&os_2); + /* this should've been enough */ + + struct istream *is = iostream_temp_finish(&os, IO_BLOCK_SIZE); + struct istream *is_2 = i_stream_create_decrypt(is, test_v1_kp.priv); + i_stream_unref(&is); + + /* read should not fail */ + while(i_stream_read_data(is_2, &ptr, &siz, 0)>0) { + test_assert(FALSE); /* should never be reached */ + }; + + test_assert(is_2->stream_errno == 0); + if (is_2->stream_errno != 0) + i_debug("error: %s", i_stream_get_error(is_2)); + i_stream_unref(&is_2); + test_end(); +} + +static +void test_write_read_v2(void) +{ + test_begin("test_write_read_v2"); + unsigned char payload[IO_BLOCK_SIZE]; + const unsigned char *ptr; + size_t pos = 0, siz; + random_fill_weak(payload, IO_BLOCK_SIZE); + + struct ostream *os = iostream_temp_create("/tmp", 0); + struct ostream *os_2 = o_stream_create_encrypt(os, "aes-256-gcm-sha256", test_v1_kp.pub, IO_STREAM_ENC_INTEGRITY_AEAD); + o_stream_nsend(os_2, payload, sizeof(payload)); + test_assert(o_stream_nfinish(os_2) == 0); + if (os_2->stream_errno != 0) + i_debug("error: %s", o_stream_get_error(os_2)); + + o_stream_unref(&os_2); + + struct istream *is = iostream_temp_finish(&os, IO_BLOCK_SIZE); + struct istream *is_2 = i_stream_create_decrypt(is, test_v1_kp.priv); + i_stream_unref(&is); + + while(i_stream_read_data(is_2, &ptr, &siz, 0)>0) { + test_assert_idx(pos + siz <= sizeof(payload), pos); + if (pos + siz > sizeof(payload)) break; + test_assert_idx(memcmp(ptr, payload + pos, siz) == 0, pos); + i_stream_skip(is_2, siz); + } + + test_assert(is_2->stream_errno == 0); + if (is_2->stream_errno != 0) + i_debug("error: %s", i_stream_get_error(is_2)); + + i_stream_unref(&is_2); + + test_end(); +} + +static +void test_write_read_v2_short(void) +{ + test_begin("test_write_read_v2_short"); + unsigned char payload[1]; + const unsigned char *ptr; + size_t pos = 0, siz; + random_fill_weak(payload, 1); + + struct ostream *os = iostream_temp_create("/tmp", 0); + struct ostream *os_2 = o_stream_create_encrypt(os, "aes-256-gcm-sha256", test_v1_kp.pub, IO_STREAM_ENC_INTEGRITY_AEAD); + o_stream_nsend(os_2, payload, sizeof(payload)); + test_assert(o_stream_nfinish(os_2) == 0); + if (os_2->stream_errno != 0) + i_debug("error: %s", o_stream_get_error(os_2)); + + o_stream_unref(&os_2); + + struct istream *is = iostream_temp_finish(&os, IO_BLOCK_SIZE); + struct istream *is_2 = i_stream_create_decrypt(is, test_v1_kp.priv); + i_stream_unref(&is); + + while(i_stream_read_data(is_2, &ptr, &siz, 0)>0) { + test_assert_idx(pos + siz <= sizeof(payload), pos); + if (pos + siz > sizeof(payload)) break; + test_assert_idx(memcmp(ptr, payload + pos, siz) == 0, pos); + i_stream_skip(is_2, siz); + } + + test_assert(is_2->stream_errno == 0); + if (is_2->stream_errno != 0) + i_debug("error: %s", i_stream_get_error(is_2)); + + i_stream_unref(&is_2); + + test_end(); +} + +static +void test_write_read_v2_empty(void) +{ + const unsigned char *ptr; + size_t siz; + test_begin("test_write_read_v2_empty"); + struct ostream *os = iostream_temp_create("/tmp", 0); + struct ostream *os_2 = o_stream_create_encrypt(os, "aes-256-gcm-sha256", test_v1_kp.pub, IO_STREAM_ENC_INTEGRITY_AEAD); + test_assert(o_stream_nfinish(os_2) == 0); + if (os_2->stream_errno != 0) + i_debug("error: %s", o_stream_get_error(os_2)); + + o_stream_unref(&os_2); + /* this should've been enough */ + + struct istream *is = iostream_temp_finish(&os, IO_BLOCK_SIZE); + struct istream *is_2 = i_stream_create_decrypt(is, test_v1_kp.priv); + i_stream_unref(&is); + + /* read should not fail */ + while(i_stream_read_data(is_2, &ptr, &siz, 0)>0) { + test_assert(FALSE); /* should never be reached */ + }; + + test_assert(is_2->stream_errno == 0); + if (is_2->stream_errno != 0) + i_debug("error: %s", i_stream_get_error(is_2)); + i_stream_unref(&is_2); + test_end(); +} + +static +void test_free_keys() { + dcrypt_key_free_private(&test_v1_kp.priv); + dcrypt_key_free_public(&test_v1_kp.pub); + dcrypt_key_free_private(&test_v2_kp.priv); + dcrypt_key_free_public(&test_v2_kp.pub); +} + +int main(void) { + dcrypt_initialize("openssl", NULL, NULL); + random_init(); + + test_assert(dcrypt_key_load_private(&test_v1_kp.priv, DCRYPT_FORMAT_PEM, key_v1_priv, NULL, NULL, NULL)); + test_assert(dcrypt_key_load_public(&test_v1_kp.pub, DCRYPT_FORMAT_PEM, key_v1_pub, NULL)); + test_assert(dcrypt_key_load_private(&test_v2_kp.priv, DCRYPT_FORMAT_PEM, key_v2_priv, NULL, NULL, NULL)); + test_assert(dcrypt_key_load_public(&test_v2_kp.pub, DCRYPT_FORMAT_PEM, key_v2_pub, NULL)); + + static void (*test_functions[])(void) = { + test_static_v1_input, + test_static_v1_input_short, + test_static_v2_input, + test_write_read_v1, + test_write_read_v1_short, + test_write_read_v1_empty, + test_write_read_v2, + test_write_read_v2_short, + test_write_read_v2_empty, + test_free_keys, + NULL + }; + + return test_run(test_functions); +} diff --git a/src/lib-dict-extra/Makefile.am b/src/lib-dict-extra/Makefile.am new file mode 100644 index 00000000000..e39d112ad8d --- /dev/null +++ b/src/lib-dict-extra/Makefile.am @@ -0,0 +1,15 @@ +noinst_LTLIBRARIES = libdict_extra.la + +dict_drivers = @dict_drivers@ + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-dict \ + -I$(top_srcdir)/src/lib-fs \ + -I$(top_srcdir)/src/lib-settings + +libdict_extra_la_SOURCES = \ + dict-fs.c \ + dict-register.c + +NOPLUGIN_LDFLAGS = diff --git a/src/lib-dict/dict-fs.c b/src/lib-dict-extra/dict-fs.c similarity index 99% rename from src/lib-dict/dict-fs.c rename to src/lib-dict-extra/dict-fs.c index ac951b9f579..3b4b58a79ed 100644 --- a/src/lib-dict/dict-fs.c +++ b/src/lib-dict-extra/dict-fs.c @@ -285,6 +285,7 @@ struct dict dict_driver_fs = { dict_transaction_memory_unset, NULL, NULL, + NULL, NULL } }; diff --git a/src/lib-dict/dict-register.c b/src/lib-dict-extra/dict-register.c similarity index 100% rename from src/lib-dict/dict-register.c rename to src/lib-dict-extra/dict-register.c diff --git a/src/lib-dict/Makefile.am b/src/lib-dict/Makefile.am index ddd4a130ce8..805ecc6a7cb 100644 --- a/src/lib-dict/Makefile.am +++ b/src/lib-dict/Makefile.am @@ -1,13 +1,9 @@ noinst_LTLIBRARIES = libdict.la noinst_LIBRARIES = libdict_backend.a -dict_drivers = @dict_drivers@ - AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ -I$(top_srcdir)/src/lib-test \ - -I$(top_srcdir)/src/lib-fs \ - -I$(top_srcdir)/src/lib-ldap \ -I$(top_srcdir)/src/lib-sql \ -I$(top_srcdir)/src/lib-settings \ $(SQL_CFLAGS) @@ -16,11 +12,9 @@ base_sources = \ dict.c \ dict-client.c \ dict-file.c \ - dict-fs.c \ dict-memcached.c \ dict-memcached-ascii.c \ dict-redis.c \ - dict-register.c \ dict-transaction-memory.c libdict_la_SOURCES = \ @@ -35,21 +29,9 @@ libdict_backend_a_SOURCES = \ nodist_libdict_backend_a_SOURCES = \ dict-drivers-register.c -NOPLUGIN_LDFLAGS = -libdict_ldap_la_LDFLAGS = -module -avoid-version $(LIBDOVECOT_LDAP) - -module_dictdir = $(moduledir)/dict -module_dict_LTLIBRARIES = \ - libdict_ldap.la - -libdict_ldap_la_SOURCES = \ - dict-ldap.c \ - dict-ldap-settings.c - headers = \ dict.h \ dict-client.h \ - dict-ldap-settings.h \ dict-private.h \ dict-sql.h \ dict-sql-settings.h \ diff --git a/src/lib-dict/dict-cdb.c b/src/lib-dict/dict-cdb.c index 755dbde5622..25e1def4e03 100644 --- a/src/lib-dict/dict-cdb.c +++ b/src/lib-dict/dict-cdb.c @@ -122,20 +122,9 @@ static int cdb_dict_lookup(struct dict *_dict, pool_t pool, struct dict dict_driver_cdb = { .name = "cdb", { - cdb_dict_init, - cdb_dict_deinit, - NULL, - cdb_dict_lookup, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL + .init = cdb_dict_init, + .deinit = cdb_dict_deinit, + .lookup = cdb_dict_lookup, } }; #endif diff --git a/src/lib-dict/dict-client.c b/src/lib-dict/dict-client.c index 5eed547a4de..916504a927f 100644 --- a/src/lib-dict/dict-client.c +++ b/src/lib-dict/dict-client.c @@ -1,10 +1,12 @@ /* Copyright (c) 2005-2016 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "array.h" #include "llist.h" #include "str.h" -#include "net.h" -#include "istream.h" +#include "strescape.h" +#include "time-util.h" +#include "connection.h" #include "ostream.h" #include "eacces-error.h" #include "dict-private.h" @@ -18,529 +20,501 @@ handle only one client at a time. This is why the default timeout is zero, so that there won't be many dict processes just doing nothing. Zero means that the socket is disconnected immediately after returning to ioloop. */ -#define DICT_CLIENT_TIMEOUT_MSECS 0 +#define DICT_CLIENT_DEFAULT_TIMEOUT_MSECS 0 /* Abort dict lookup after this many seconds. */ -#define DICT_CLIENT_READ_TIMEOUT_SECS 30 -/* Log a warning if dict lookup takes longer than this many seconds. */ -#define DICT_CLIENT_READ_WARN_TIMEOUT_SECS 5 +#define DICT_CLIENT_REQUEST_TIMEOUT_MSECS 30000 +/* Log a warning if dict lookup takes longer than this many milliseconds. */ +#define DICT_CLIENT_REQUEST_WARN_TIMEOUT_MSECS 5000 + +struct client_dict_cmd { + int refcount; + struct client_dict *dict; + struct timeval start_time; + char *query; + + bool retry_errors; + bool no_replies; + bool unfinished; + + void (*callback)(struct client_dict_cmd *cmd, + const char *line, const char *error); + struct client_dict_iterate_context *iter; + + struct { + dict_lookup_callback_t *lookup; + dict_transaction_commit_callback_t *commit; + void *context; + } api_callback; +}; + +struct dict_connection { + struct connection conn; + struct client_dict *dict; +}; struct client_dict { struct dict dict; + struct dict_connection conn; - pool_t pool; - int fd; - const char *uri; - const char *username; - const char *path; + char *uri, *username; enum dict_data_type value_type; time_t last_failed_connect; - struct istream *input; - struct ostream *output; - struct io *io; + struct ioloop *ioloop, *prev_ioloop; + struct timeout *to_requests; struct timeout *to_idle; + unsigned int idle_msecs; + struct timeval last_input; + ARRAY(struct client_dict_cmd *) cmds; struct client_dict_transaction_context *transactions; - unsigned int connect_counter; unsigned int transaction_id_counter; - unsigned int async_commits; - unsigned int iter_replies_skip; +}; - unsigned int in_iteration:1; - unsigned int handshaked:1; +struct client_dict_iter_result { + const char *key, *value; }; struct client_dict_iterate_context { struct dict_iterate_context ctx; + char *error; + + pool_t results_pool; + ARRAY(struct client_dict_iter_result) results; + unsigned int result_idx; - pool_t pool; - bool failed; + bool async; bool finished; + bool deinit; }; struct client_dict_transaction_context { struct dict_transaction_context ctx; struct client_dict_transaction_context *prev, *next; - /* for async commits */ - dict_transaction_commit_callback_t *callback; - void *context; + char *error; unsigned int id; - unsigned int connect_counter; - unsigned int failed:1; - unsigned int sent_begin:1; - unsigned int async:1; - unsigned int committed:1; + bool sent_begin:1; }; -static int client_dict_connect(struct client_dict *dict); -static void client_dict_disconnect(struct client_dict *dict); - -const char *dict_client_escape(const char *src) -{ - const char *p; - string_t *dest; - - /* first do a quick lookup to see if there's anything to escape. - probably not. */ - for (p = src; *p != '\0'; p++) { - if (*p == '\t' || *p == '\n' || *p == '\001') - break; - } - - if (*p == '\0') - return src; - - dest = t_str_new(256); - str_append_n(dest, src, p - src); - - for (; *p != '\0'; p++) { - switch (*p) { - case '\t': - str_append_c(dest, '\001'); - str_append_c(dest, 't'); - break; - case '\n': - str_append_c(dest, '\001'); - str_append_c(dest, 'n'); - break; - case '\001': - str_append_c(dest, '\001'); - str_append_c(dest, '1'); - break; - default: - str_append_c(dest, *p); - break; - } - } - return str_c(dest); -} - -const char *dict_client_unescape(const char *src) -{ - const char *p; - string_t *dest; - - /* first do a quick lookup to see if there's anything to unescape. - probably not. */ - for (p = src; *p != '\0'; p++) { - if (*p == '\001') - break; - } - - if (*p == '\0') - return src; - - dest = t_str_new(256); - str_append_n(dest, src, p - src); - for (; *p != '\0'; p++) { - if (*p != '\001') - str_append_c(dest, *p); - else if (p[1] != '\0') { - p++; - switch (*p) { - case '1': - str_append_c(dest, '\001'); - break; - case 't': - str_append_c(dest, '\t'); - break; - case 'n': - str_append_c(dest, '\n'); - break; - } - } - } - return str_c(dest); -} - -static int client_dict_send_query(struct client_dict *dict, const char *query) -{ - if (dict->output == NULL) { - /* not connected currently */ - if (client_dict_connect(dict) < 0) - return -1; - } - - if (o_stream_send_str(dict->output, query) < 0 || - o_stream_flush(dict->output) < 0) { - /* Send failed */ - if (!dict->handshaked) { - /* we're trying to send hello, don't try to reconnect */ - return -1; - } +static struct connection_list *dict_connections; - /* Reconnect and try again. */ - client_dict_disconnect(dict); - if (client_dict_connect(dict) < 0) - return -1; +static int client_dict_connect(struct client_dict *dict, const char **error_r); +static void client_dict_disconnect(struct client_dict *dict, const char *reason); - if (o_stream_send_str(dict->output, query) < 0 || - o_stream_flush(dict->output) < 0) { - i_error("write(%s) failed: %m", dict->path); - client_dict_disconnect(dict); - return -1; - } - } - return 0; +static struct client_dict_cmd * +client_dict_cmd_init(struct client_dict *dict, const char *query) +{ + struct client_dict_cmd *cmd; + + cmd = i_new(struct client_dict_cmd, 1); + cmd->refcount = 1; + cmd->dict = dict; + cmd->query = i_strdup(query); + cmd->start_time = ioloop_timeval; + return cmd; } -static int -client_dict_transaction_send_begin(struct client_dict_transaction_context *ctx) +static void client_dict_cmd_ref(struct client_dict_cmd *cmd) { - struct client_dict *dict = (struct client_dict *)ctx->ctx.dict; - - if (ctx->failed) - return -1; - - T_BEGIN { - const char *query; - - query = t_strdup_printf("%c%u\n", DICT_PROTOCOL_CMD_BEGIN, - ctx->id); - if (client_dict_send_query(dict, query) < 0) - ctx->failed = TRUE; - else - ctx->connect_counter = dict->connect_counter; - } T_END; - - return ctx->failed ? -1 : 0; + i_assert(cmd->refcount > 0); + cmd->refcount++; } -static int ATTR_NOWARN_UNUSED_RESULT -client_dict_send_transaction_query(struct client_dict_transaction_context *ctx, - const char *query) +static bool client_dict_cmd_unref(struct client_dict_cmd *cmd) { - struct client_dict *dict = (struct client_dict *)ctx->ctx.dict; + i_assert(cmd->refcount > 0); + if (--cmd->refcount > 0) + return TRUE; - if (!ctx->sent_begin) { - if (client_dict_transaction_send_begin(ctx) < 0) - return -1; - ctx->sent_begin = TRUE; - } - - if (ctx->connect_counter != dict->connect_counter || ctx->failed) - return -1; + i_free(cmd->query); + i_free(cmd); + return FALSE; +} - if (dict->output == NULL) { - /* not connected, this'll fail */ - return -1; +static void dict_pre_api_callback(struct client_dict *dict) +{ + if (dict->prev_ioloop != NULL) { + /* Don't let callback see that we've created our + internal ioloop in case it wants to add some ios + or timeouts. */ + current_ioloop = dict->prev_ioloop; } +} - if (o_stream_send_str(dict->output, query) < 0 || - o_stream_flush(dict->output) < 0) { - /* Send failed. Our transactions have died, so don't even try - to re-send the command */ - ctx->failed = TRUE; - client_dict_disconnect(dict); - return -1; +static void dict_post_api_callback(struct client_dict *dict) +{ + if (dict->prev_ioloop != NULL) { + current_ioloop = dict->ioloop; + /* stop client_dict_wait() */ + io_loop_stop(dict->ioloop); } - return 0; } -static struct client_dict_transaction_context * -client_dict_transaction_find(struct client_dict *dict, unsigned int id) +static bool +dict_cmd_callback_line(struct client_dict_cmd *cmd, const char *line) { - struct client_dict_transaction_context *ctx; - - for (ctx = dict->transactions; ctx != NULL; ctx = ctx->next) { - if (ctx->id == id) - return ctx; - } - return NULL; + cmd->unfinished = FALSE; + cmd->callback(cmd, line, NULL); + return !cmd->unfinished; } static void -client_dict_finish_transaction(struct client_dict *dict, - unsigned int id, int ret) +dict_cmd_callback_error(struct client_dict_cmd *cmd, const char *error) { - struct client_dict_transaction_context *ctx; - - ctx = client_dict_transaction_find(dict, id); - if (ctx == NULL) { - i_error("dict-client: Unknown transaction id %u", id); - return; - } - ctx->failed = TRUE; - if (!ctx->committed) - return; + cmd->unfinished = FALSE; + if (cmd->callback != NULL) + cmd->callback(cmd, NULL, error); + i_assert(!cmd->unfinished); +} - /* the callback may call the dict code again, so remove this - transaction before calling it */ - i_assert(dict->async_commits > 0); - if (--dict->async_commits == 0) { - if (dict->io != NULL) - io_remove(&dict->io); - } - DLLIST_REMOVE(&dict->transactions, ctx); +static void client_dict_input_timeout(struct client_dict *dict) +{ + int diff = timeval_diff_msecs(&ioloop_timeval, &dict->last_input); - if (ctx->callback != NULL) - ctx->callback(ret, ctx->context); - i_free(ctx); + client_dict_disconnect(dict, t_strdup_printf( + "Timeout: No input from dict for %u.%03u secs", + diff/1000, diff%1000)); } -static ssize_t client_dict_read_timeout(struct client_dict *dict) +static int +client_dict_cmd_query_send(struct client_dict *dict, const char *query) { - time_t now, timeout; - unsigned int diff; + struct const_iovec iov[2]; ssize_t ret; - now = time(NULL); - timeout = now + DICT_CLIENT_READ_TIMEOUT_SECS; - - do { - alarm(timeout - now); - ret = i_stream_read(dict->input); - alarm(0); - if (ret != 0) - break; - - /* interrupted most likely because of timeout, - but check anyway. */ - now = time(NULL); - } while (now < timeout); - - if (ret > 0) { - diff = time(NULL) - now; - if (diff >= DICT_CLIENT_READ_WARN_TIMEOUT_SECS) { - i_warning("read(%s): dict lookup took %u seconds", - dict->path, diff); - } - } - return ret; + iov[0].iov_base = query; + iov[0].iov_len = strlen(query); + iov[1].iov_base = "\n"; + iov[1].iov_len = 1; + ret = o_stream_sendv(dict->conn.conn.output, iov, 2); + if (ret < 0) + return -1; + i_assert((size_t)ret == iov[0].iov_len + 1); + return 0; } -static int -client_dict_read_one_line_real(struct client_dict *dict, char **line_r) +static bool +client_dict_cmd_send(struct client_dict *dict, struct client_dict_cmd **_cmd, + const char **error_r) { - unsigned int id; - char *line; - ssize_t ret; + struct client_dict_cmd *cmd = *_cmd; + const char *error = NULL; + bool retry = cmd->retry_errors; + int ret; - *line_r = NULL; - while ((line = i_stream_next_line(dict->input)) == NULL) { - ret = client_dict_read_timeout(dict); - switch (ret) { - case -1: - if (dict->input->stream_errno != 0) - i_error("read(%s) failed: %m", dict->path); - else { - i_error("read(%s) failed: Remote disconnected", - dict->path); - } - return -1; - case -2: - i_error("read(%s) returned too much data", dict->path); - return -1; - case 0: - i_error("read(%s) failed: Timeout after %u seconds", - dict->path, DICT_CLIENT_READ_TIMEOUT_SECS); - return -1; - default: - i_assert(ret > 0); - break; + *_cmd = NULL; + + /* we're no longer idling. even with no_replies=TRUE we're going to + wait for COMMIT/ROLLBACK. */ + if (dict->to_idle != NULL) + timeout_remove(&dict->to_idle); + + if (client_dict_connect(dict, &error) < 0) { + retry = FALSE; + ret = -1; + } else { + ret = client_dict_cmd_query_send(dict, cmd->query); + if (ret < 0) { + error = t_strdup_printf("write(%s) failed: %s", dict->conn.conn.name, + o_stream_get_error(dict->conn.conn.output)); } } - if (*line == DICT_PROTOCOL_REPLY_ASYNC_COMMIT) { - switch (line[1]) { - case DICT_PROTOCOL_REPLY_OK: - ret = 1; - break; - case DICT_PROTOCOL_REPLY_NOTFOUND: + if (ret < 0 && retry) { + /* Reconnect and try again. */ + client_dict_disconnect(dict, error); + if (client_dict_connect(dict, &error) < 0) + ; + else if (client_dict_cmd_query_send(dict, cmd->query) < 0) { + error = t_strdup_printf("write(%s) failed: %s", dict->conn.conn.name, + o_stream_get_error(dict->conn.conn.output)); + } else { ret = 0; - break; - case DICT_PROTOCOL_REPLY_FAIL: - ret = -1; - break; - default: - i_error("dict-client: Invalid async commit line: %s", - line); - return -1; - } - if (str_to_uint(line+2, &id) < 0) { - i_error("dict-client: Invalid ID"); - return -1; } - client_dict_finish_transaction(dict, id, ret); - return 0; } - if (dict->iter_replies_skip > 0) { - /* called aborted the iteration before finishing it. - skip over the iteration reply */ - if (*line == DICT_PROTOCOL_REPLY_OK) - return 0; - if (*line != '\0' && *line != DICT_PROTOCOL_REPLY_FAIL) { - i_error("dict-client: Invalid iteration reply line: %s", - line); - return -1; + + if (cmd->no_replies) { + /* just send and forget */ + client_dict_cmd_unref(cmd); + return TRUE; + } else if (ret < 0) { + i_assert(error != NULL); + dict_cmd_callback_error(cmd, error); + client_dict_cmd_unref(cmd); + if (error_r != NULL) + *error_r = error; + return FALSE; + } else { + if (dict->to_requests == NULL) { + dict->to_requests = + timeout_add(DICT_CLIENT_REQUEST_TIMEOUT_MSECS, + client_dict_input_timeout, dict); } - dict->iter_replies_skip--; - return 0; + array_append(&dict->cmds, &cmd, 1); + return TRUE; } - *line_r = line; - return 1; } -static int client_dict_read_one_line(struct client_dict *dict, char **line_r) +static void +client_dict_transaction_send_begin(struct client_dict_transaction_context *ctx) { - int ret; + struct client_dict *dict = (struct client_dict *)ctx->ctx.dict; + struct client_dict_cmd *cmd; + const char *query, *error; - if ((ret = client_dict_read_one_line_real(dict, line_r)) < 0) - client_dict_disconnect(dict); - return ret; + i_assert(ctx->error == NULL); + + ctx->sent_begin = TRUE; + + /* transactions commands don't have replies. only COMMIT has. */ + query = t_strdup_printf("%c%u", DICT_PROTOCOL_CMD_BEGIN, ctx->id); + cmd = client_dict_cmd_init(dict, query); + cmd->no_replies = TRUE; + cmd->retry_errors = TRUE; + if (!client_dict_cmd_send(dict, &cmd, &error)) + ctx->error = i_strdup(error); +} + +static void +client_dict_send_transaction_query(struct client_dict_transaction_context *ctx, + const char *query) +{ + struct client_dict *dict = (struct client_dict *)ctx->ctx.dict; + struct client_dict_cmd *cmd; + const char *error; + + if (ctx->error != NULL) + return; + + if (!ctx->sent_begin) + client_dict_transaction_send_begin(ctx); + + cmd = client_dict_cmd_init(dict, query); + cmd->no_replies = TRUE; + if (!client_dict_cmd_send(dict, &cmd, &error)) + ctx->error = i_strdup(error); } static bool client_dict_is_finished(struct client_dict *dict) { - return dict->transactions == NULL && !dict->in_iteration && - dict->async_commits == 0; + return dict->transactions == NULL && array_count(&dict->cmds) == 0; } static void client_dict_timeout(struct client_dict *dict) { if (client_dict_is_finished(dict)) - client_dict_disconnect(dict); + client_dict_disconnect(dict, "Idle disconnection"); } static void client_dict_add_timeout(struct client_dict *dict) { if (dict->to_idle != NULL) { -#if DICT_CLIENT_TIMEOUT_MSECS > 0 - timeout_reset(dict->to_idle); -#endif + if (dict->idle_msecs > 0) + timeout_reset(dict->to_idle); } else if (client_dict_is_finished(dict)) { - dict->to_idle = timeout_add(DICT_CLIENT_TIMEOUT_MSECS, + dict->to_idle = timeout_add(dict->idle_msecs, client_dict_timeout, dict); + if (dict->to_requests != NULL) + timeout_remove(&dict->to_requests); } } -static char *client_dict_read_line(struct client_dict *dict) +static int dict_conn_input_line(struct connection *_conn, const char *line) { - char *line; + struct dict_connection *conn = (struct dict_connection *)_conn; + struct client_dict *dict = conn->dict; + struct client_dict_cmd *const *cmds; + unsigned int count; + bool finished; + int diff; + + dict->last_input = ioloop_timeval; + if (dict->to_requests != NULL) + timeout_reset(dict->to_requests); - while (client_dict_read_one_line(dict, &line) == 0) - ; + cmds = array_get(&conn->dict->cmds, &count); + if (count == 0) { + i_error("%s: Received reply without pending commands: %s", + dict->conn.conn.name, line); + return -1; + } + i_assert(!cmds[0]->no_replies); + + client_dict_cmd_ref(cmds[0]); + finished = dict_cmd_callback_line(cmds[0], line); + if (!client_dict_cmd_unref(cmds[0])) { + /* disconnected during command handling */ + return -1; + } + if (!finished) { + /* more lines needed for this command */ + return 1; + } + diff = timeval_diff_msecs(&ioloop_timeval, &cmds[0]->start_time); + if (diff >= DICT_CLIENT_REQUEST_WARN_TIMEOUT_MSECS) { + i_warning("read(%s): dict lookup took %u.%03u seconds: %s", + dict->conn.conn.name, diff/1000, diff % 1000, + cmds[0]->query); + } + client_dict_cmd_unref(cmds[0]); + array_delete(&dict->cmds, 0, 1); client_dict_add_timeout(dict); - return line; + return 1; } -static int client_dict_connect(struct client_dict *dict) +static int client_dict_connect(struct client_dict *dict, const char **error_r) { const char *query; + if (dict->conn.conn.fd_in != -1) + return 0; if (dict->last_failed_connect == ioloop_time) { /* Try again later */ + *error_r = "Waiting until the next connect attempt"; return -1; } - dict->fd = net_connect_unix(dict->path); - if (dict->fd == -1) { + if (connection_client_connect(&dict->conn.conn) < 0) { dict->last_failed_connect = ioloop_time; if (errno == EACCES) { - i_error("%s", eacces_error_get("net_connect_unix", - dict->path)); + *error_r = eacces_error_get("net_connect_unix", + dict->conn.conn.name); } else { - i_error("net_connect_unix(%s) failed: %m", - dict->path); + *error_r = t_strdup_printf( + "net_connect_unix(%s) failed: %m", dict->conn.conn.name); } return -1; } - /* Dictionary lookups are blocking */ - net_set_nonblock(dict->fd, FALSE); - - dict->input = i_stream_create_fd(dict->fd, (size_t)-1, FALSE); - dict->output = o_stream_create_fd(dict->fd, 4096, FALSE); - query = t_strdup_printf("%c%u\t%u\t%d\t%s\t%s\n", DICT_PROTOCOL_CMD_HELLO, DICT_CLIENT_PROTOCOL_MAJOR_VERSION, DICT_CLIENT_PROTOCOL_MINOR_VERSION, dict->value_type, dict->username, dict->uri); - if (client_dict_send_query(dict, query) < 0) { - dict->last_failed_connect = ioloop_time; - client_dict_disconnect(dict); - return -1; - } - - dict->handshaked = TRUE; + o_stream_nsend_str(dict->conn.conn.output, query); + client_dict_add_timeout(dict); return 0; } -static void client_dict_disconnect(struct client_dict *dict) +static void +client_dict_abort_commands(struct client_dict *dict, const char *reason) +{ + ARRAY(struct client_dict_cmd *) cmds_copy; + struct client_dict_cmd *const *cmdp; + + /* abort all commands */ + t_array_init(&cmds_copy, array_count(&dict->cmds)); + array_append_array(&cmds_copy, &dict->cmds); + array_clear(&dict->cmds); + + array_foreach(&cmds_copy, cmdp) { + dict_cmd_callback_error(*cmdp, reason); + client_dict_cmd_unref(*cmdp); + } +} + +static void client_dict_disconnect(struct client_dict *dict, const char *reason) { struct client_dict_transaction_context *ctx, *next; - dict->connect_counter++; - dict->handshaked = FALSE; - dict->iter_replies_skip = 0; + client_dict_abort_commands(dict, reason); - /* abort all pending async commits */ + /* all transactions that have sent BEGIN are no longer valid */ for (ctx = dict->transactions; ctx != NULL; ctx = next) { next = ctx->next; - if (ctx->async) - client_dict_finish_transaction(dict, ctx->id, -1); + if (ctx->sent_begin && ctx->error == NULL) + ctx->error = i_strdup(reason); } if (dict->to_idle != NULL) timeout_remove(&dict->to_idle); - if (dict->io != NULL) - io_remove(&dict->io); - if (dict->input != NULL) - i_stream_destroy(&dict->input); - if (dict->output != NULL) - o_stream_destroy(&dict->output); + if (dict->to_requests != NULL) + timeout_remove(&dict->to_requests); + connection_disconnect(&dict->conn.conn); +} - if (dict->fd != -1) { - if (close(dict->fd) < 0) - i_error("close(%s) failed: %m", dict->path); - dict->fd = -1; - } +static void dict_conn_destroy(struct connection *_conn) +{ + struct dict_connection *conn = (struct dict_connection *)_conn; + + client_dict_disconnect(conn->dict, connection_disconnect_reason(_conn)); } +static const struct connection_settings dict_conn_set = { + .input_max_size = (size_t)-1, + .output_max_size = (size_t)-1, + .client = TRUE +}; + +static const struct connection_vfuncs dict_conn_vfuncs = { + .destroy = dict_conn_destroy, + .input_line = dict_conn_input_line +}; + static int client_dict_init(struct dict *driver, const char *uri, const struct dict_settings *set, struct dict **dict_r, const char **error_r) { + struct ioloop *old_ioloop = current_ioloop; struct client_dict *dict; - const char *dest_uri; - pool_t pool; - - /* uri = [] ":" */ + const char *p, *dest_uri, *path; + unsigned int idle_msecs = DICT_CLIENT_DEFAULT_TIMEOUT_MSECS; + + /* uri = [idle_msecs=:] [] ":" */ + if (strncmp(uri, "idle_msecs=", 11) == 0) { + p = strchr(uri+11, ':'); + if (p == NULL) { + *error_r = t_strdup_printf("Invalid URI: %s", uri); + return -1; + } + if (str_to_uint(t_strdup_until(uri+11, p), &idle_msecs) < 0) { + *error_r = "Invalid idle_msecs"; + return -1; + } + uri = p+1; + } dest_uri = strchr(uri, ':'); if (dest_uri == NULL) { *error_r = t_strdup_printf("Invalid URI: %s", uri); return -1; } - pool = pool_alloconly_create("client dict", 1024); - dict = p_new(pool, struct client_dict, 1); - dict->pool = pool; + if (dict_connections == NULL) { + dict_connections = connection_list_init(&dict_conn_set, + &dict_conn_vfuncs); + } + + dict = i_new(struct client_dict, 1); dict->dict = *driver; + dict->conn.dict = dict; dict->value_type = set->value_type; - dict->username = p_strdup(pool, set->username); - - dict->fd = -1; + dict->username = i_strdup(set->username); + dict->idle_msecs = idle_msecs; + i_array_init(&dict->cmds, 32); if (uri[0] == ':') { /* default path */ - dict->path = p_strconcat(pool, set->base_dir, - "/"DEFAULT_DICT_SERVER_SOCKET_FNAME, NULL); + path = t_strconcat(set->base_dir, + "/"DEFAULT_DICT_SERVER_SOCKET_FNAME, NULL); } else if (uri[0] == '/') { /* absolute path */ - dict->path = p_strdup_until(pool, uri, dest_uri); + path = t_strdup_until(uri, dest_uri); } else { /* relative path to base_dir */ - dict->path = p_strconcat(pool, set->base_dir, "/", - p_strdup_until(pool, uri, dest_uri), NULL); + path = t_strconcat(set->base_dir, "/", + t_strdup_until(uri, dest_uri), NULL); } - dict->uri = p_strdup(pool, dest_uri + 1); + connection_init_client_unix(dict_connections, &dict->conn.conn, path); + dict->uri = i_strdup(dest_uri + 1); + + dict->ioloop = io_loop_create(); + io_loop_set_current(old_ioloop); *dict_r = &dict->dict; return 0; } @@ -548,154 +522,300 @@ client_dict_init(struct dict *driver, const char *uri, static void client_dict_deinit(struct dict *_dict) { struct client_dict *dict = (struct client_dict *)_dict; + struct ioloop *old_ioloop = current_ioloop; + + client_dict_disconnect(dict, "Deinit"); + connection_deinit(&dict->conn.conn); - client_dict_disconnect(dict); i_assert(dict->transactions == NULL); - pool_unref(&dict->pool); + i_assert(array_count(&dict->cmds) == 0); + + io_loop_set_current(dict->ioloop); + io_loop_destroy(&dict->ioloop); + io_loop_set_current(old_ioloop); + + array_free(&dict->cmds); + i_free(dict->username); + i_free(dict->uri); + i_free(dict); + + if (dict_connections->connections == NULL) + connection_list_deinit(&dict_connections); } static int client_dict_wait(struct dict *_dict) { struct client_dict *dict = (struct client_dict *)_dict; - char *line; - int ret; - if (!dict->handshaked) - return -1; + if (array_count(&dict->cmds) == 0) + return 0; - while (dict->async_commits > 0) { - if ((ret = client_dict_read_one_line(dict, &line)) < 0) - return -1; + dict->prev_ioloop = current_ioloop; + io_loop_set_current(dict->ioloop); + dict_switch_ioloop(_dict); + while (array_count(&dict->cmds) > 0) + io_loop_run(dict->ioloop); - if (ret > 0) { - i_error("dict-client: Unexpected reply waiting waiting for async commits: %s", line); - client_dict_disconnect(dict); - return -1; - } - } + io_loop_set_current(dict->prev_ioloop); + dict->prev_ioloop = NULL; + + dict_switch_ioloop(_dict); return 0; } -static int client_dict_lookup(struct dict *_dict, pool_t pool, - const char *key, const char **value_r) +static bool client_dict_switch_ioloop(struct dict *_dict) { struct client_dict *dict = (struct client_dict *)_dict; - const char *line; - int ret; - T_BEGIN { - const char *query; - - query = t_strdup_printf("%c%s\n", DICT_PROTOCOL_CMD_LOOKUP, - dict_client_escape(key)); - ret = client_dict_send_query(dict, query); - } T_END; - if (ret < 0) - return -1; - - /* read reply */ - line = client_dict_read_line(dict); - if (line == NULL) - return -1; + if (dict->to_idle != NULL) + dict->to_idle = io_loop_move_timeout(&dict->to_idle); + if (dict->to_requests != NULL) + dict->to_requests = io_loop_move_timeout(&dict->to_requests); + connection_switch_ioloop(&dict->conn.conn); + return array_count(&dict->cmds) > 0; +} - switch (*line) { +static void +client_dict_lookup_async_callback(struct client_dict_cmd *cmd, const char *line, + const char *error) +{ + struct client_dict *dict = cmd->dict; + struct dict_lookup_result result; + + memset(&result, 0, sizeof(result)); + if (error != NULL) { + result.ret = -1; + result.error = error; + } else switch (*line) { case DICT_PROTOCOL_REPLY_OK: - *value_r = p_strdup(pool, dict_client_unescape(line + 1)); - return 1; + result.value = t_str_tabunescape(line + 1); + result.ret = 1; + break; case DICT_PROTOCOL_REPLY_NOTFOUND: - *value_r = NULL; - return 0; + result.ret = 0; + break; case DICT_PROTOCOL_REPLY_FAIL: - return -1; + result.error = line[1] == '\0' ? "dict-server returned failure" : + t_strdup_printf("dict-server returned failure: %s", + t_str_tabunescape(line+1)); + result.ret = -1; + break; default: - i_error("dict-client: Invalid lookup '%s' reply: %s", key, line); - client_dict_disconnect(dict); - return -1; + result.error = t_strdup_printf( + "dict-client: Invalid lookup '%s' reply: %s", + cmd->query, line); + client_dict_disconnect(dict, result.error); + result.ret = -1; + break; } + dict_pre_api_callback(dict); + cmd->api_callback.lookup(&result, cmd->api_callback.context); + dict_post_api_callback(dict); } -static struct dict_iterate_context * -client_dict_iterate_init(struct dict *_dict, const char *const *paths, - enum dict_iterate_flags flags) +static void +client_dict_lookup_async(struct dict *_dict, const char *key, + dict_lookup_callback_t *callback, void *context) { struct client_dict *dict = (struct client_dict *)_dict; - struct client_dict_iterate_context *ctx; + struct client_dict_cmd *cmd; + const char *query; - if (dict->in_iteration) - i_panic("dict-client: Only one iteration supported"); - dict->in_iteration = TRUE; + query = t_strdup_printf("%c%s", DICT_PROTOCOL_CMD_LOOKUP, + str_tabescape(key)); + cmd = client_dict_cmd_init(dict, query); + cmd->callback = client_dict_lookup_async_callback; + cmd->api_callback.lookup = callback; + cmd->api_callback.context = context; + cmd->retry_errors = TRUE; - ctx = i_new(struct client_dict_iterate_context, 1); - ctx->ctx.dict = _dict; - ctx->pool = pool_alloconly_create("client dict iteration", 512); + client_dict_cmd_send(dict, &cmd, NULL); +} - T_BEGIN { - string_t *query = t_str_new(256); - unsigned int i; +static void client_dict_lookup_callback(const struct dict_lookup_result *result, + void *context) +{ + struct dict_lookup_result *result_copy = context; - str_printfa(query, "%c%d", DICT_PROTOCOL_CMD_ITERATE, flags); - for (i = 0; paths[i] != NULL; i++) { - str_append_c(query, '\t'); - str_append(query, dict_client_escape(paths[i])); - } - str_append_c(query, '\n'); - if (client_dict_send_query(dict, str_c(query)) < 0) - ctx->failed = TRUE; - } T_END; - return &ctx->ctx; + *result_copy = *result; } -static bool client_dict_iterate(struct dict_iterate_context *_ctx, - const char **key_r, const char **value_r) +static int client_dict_lookup(struct dict *_dict, pool_t pool, const char *key, + const char **value_r) { - struct client_dict_iterate_context *ctx = - (struct client_dict_iterate_context *)_ctx; - struct client_dict *dict = (struct client_dict *)_ctx->dict; - char *line, *key, *value; + struct dict_lookup_result result; - if (ctx->failed) - return FALSE; + memset(&result, 0, sizeof(result)); + result.ret = -2; - /* read next reply */ - line = client_dict_read_line(dict); - if (line == NULL) { - ctx->failed = TRUE; - return FALSE; - } + client_dict_lookup_async(_dict, key, client_dict_lookup_callback, &result); + if (result.ret == -2) + client_dict_wait(_dict); - if (*line == '\0') { - /* end of iteration */ - ctx->finished = TRUE; - return FALSE; + switch (result.ret) { + case -1: + i_error("dict-client: Lookup '%s' failed: %s", key, result.error); + return -1; + case 0: + *value_r = NULL; + return 0; + case 1: + *value_r = p_strdup(pool, result.value); + return 1; } + i_unreached(); +} + +static void client_dict_iterate_free(struct client_dict_iterate_context *ctx) +{ + if (!ctx->deinit || !ctx->finished) + return; + i_free(ctx->error); + i_free(ctx); +} - /* line contains key \t value */ - p_clear(ctx->pool); +static void +client_dict_iter_api_callback(struct client_dict_iterate_context *ctx, + struct client_dict *dict) +{ + if (ctx->deinit) { + /* iterator was already deinitialized */ + return; + } + if (ctx->ctx.async_callback != NULL) { + dict_pre_api_callback(dict); + ctx->ctx.async_callback(ctx->ctx.async_context); + dict_post_api_callback(dict); + } else { + /* synchronous lookup */ + io_loop_stop(dict->ioloop); + } +} - switch (*line) { +static void +client_dict_iter_async_callback(struct client_dict_cmd *cmd, const char *line, + const char *error) +{ + struct client_dict_iterate_context *ctx = cmd->iter; + struct client_dict *dict = cmd->dict; + struct client_dict_iter_result *result; + const char *key = NULL, *value = NULL; + + if (error != NULL) { + /* failed */ + } else switch (*line) { + case '\0': + /* end of iteration */ + ctx->finished = TRUE; + client_dict_iter_api_callback(ctx, dict); + client_dict_iterate_free(ctx); + return; case DICT_PROTOCOL_REPLY_OK: + /* key \t value */ key = line+1; value = strchr(key, '\t'); break; case DICT_PROTOCOL_REPLY_FAIL: - ctx->failed = TRUE; - return FALSE; + error = t_strdup_printf("dict-server returned failure: %s", line+1); + break; default: - key = NULL; - value = NULL; break; } - if (value == NULL) { + if (value == NULL && error == NULL) { /* broken protocol */ - i_error("dict client (%s) sent broken iterate reply: %s", dict->path, line); - ctx->failed = TRUE; + error = t_strdup_printf("dict client (%s) sent broken iterate reply: %s", + dict->conn.conn.name, line); + client_dict_disconnect(dict, error); + } + + if (error != NULL) { + if (ctx->error == NULL) + ctx->error = i_strdup(error); + ctx->finished = TRUE; + if (dict->prev_ioloop != NULL) { + /* stop client_dict_wait() */ + io_loop_stop(dict->ioloop); + } + client_dict_iterate_free(ctx); + return; + } + cmd->unfinished = TRUE; + + if (ctx->deinit) { + /* iterator was already deinitialized */ + return; + } + + key = t_strdup_until(key, value++); + result = array_append_space(&ctx->results); + result->key = p_strdup(ctx->results_pool, t_str_tabunescape(key)); + result->value = p_strdup(ctx->results_pool, t_str_tabunescape(value)); + + client_dict_iter_api_callback(ctx, dict); +} + +static struct dict_iterate_context * +client_dict_iterate_init(struct dict *_dict, const char *const *paths, + enum dict_iterate_flags flags) +{ + struct client_dict *dict = (struct client_dict *)_dict; + struct client_dict_iterate_context *ctx; + struct client_dict_cmd *cmd; + string_t *query = t_str_new(256); + unsigned int i; + + ctx = i_new(struct client_dict_iterate_context, 1); + ctx->ctx.dict = _dict; + ctx->results_pool = pool_alloconly_create("client dict iteration", 512); + ctx->async = (flags & DICT_ITERATE_FLAG_ASYNC) != 0; + i_array_init(&ctx->results, 64); + + str_printfa(query, "%c%d", DICT_PROTOCOL_CMD_ITERATE, flags); + for (i = 0; paths[i] != NULL; i++) { + str_append_c(query, '\t'); + str_append(query, str_tabescape(paths[i])); + } + + cmd = client_dict_cmd_init(dict, str_c(query)); + cmd->iter = ctx; + cmd->callback = client_dict_iter_async_callback; + cmd->retry_errors = TRUE; + + client_dict_cmd_send(dict, &cmd, NULL); + return &ctx->ctx; +} + +static bool client_dict_iterate(struct dict_iterate_context *_ctx, + const char **key_r, const char **value_r) +{ + struct client_dict_iterate_context *ctx = + (struct client_dict_iterate_context *)_ctx; + const struct client_dict_iter_result *results; + unsigned int count; + + if (ctx->error != NULL) { + ctx->ctx.has_more = FALSE; return FALSE; } - *value++ = '\0'; - *key_r = p_strdup(ctx->pool, dict_client_unescape(key)); - *value_r = p_strdup(ctx->pool, dict_client_unescape(value)); - return TRUE; + results = array_get(&ctx->results, &count); + if (ctx->result_idx < count) { + *key_r = results[ctx->result_idx].key; + *value_r = results[ctx->result_idx].value; + ctx->ctx.has_more = TRUE; + ctx->result_idx++; + return TRUE; + } + ctx->ctx.has_more = !ctx->finished; + ctx->result_idx = 0; + array_clear(&ctx->results); + p_clear(ctx->results_pool); + + if (!ctx->async && ctx->ctx.has_more) { + client_dict_wait(_ctx->dict); + return client_dict_iterate(_ctx, key_r, value_r); + } + return FALSE; } static int client_dict_iterate_deinit(struct dict_iterate_context *_ctx) @@ -703,14 +823,15 @@ static int client_dict_iterate_deinit(struct dict_iterate_context *_ctx) struct client_dict *dict = (struct client_dict *)_ctx->dict; struct client_dict_iterate_context *ctx = (struct client_dict_iterate_context *)_ctx; - int ret = ctx->failed ? -1 : 0; + int ret = ctx->error != NULL ? -1 : 0; - if (!ctx->finished) - dict->iter_replies_skip++; + ctx->deinit = TRUE; - pool_unref(&ctx->pool); - i_free(ctx); - dict->in_iteration = FALSE; + if (ret < 0) + i_error("dict-client: Iteration failed: %s", ctx->error); + array_free(&ctx->results); + pool_unref(&ctx->results_pool); + client_dict_iterate_free(ctx); client_dict_add_timeout(dict); return ret; @@ -730,23 +851,46 @@ client_dict_transaction_init(struct dict *_dict) return &ctx->ctx; } -static void dict_async_input(struct client_dict *dict) +static void +client_dict_transaction_commit_callback(struct client_dict_cmd *cmd, + const char *line, const char *error) { - char *line; - int ret; - - i_assert(!dict->in_iteration); + struct client_dict *dict = cmd->dict; + int ret = -1; - do { - ret = client_dict_read_one_line(dict, &line); - } while (ret == 0 && i_stream_get_data_size(dict->input) > 0); + if (error != NULL) { + /* failed */ + i_error("dict-client: Commit failed: %s", error); + } else switch (*line) { + case DICT_PROTOCOL_REPLY_OK: + ret = 1; + break; + case DICT_PROTOCOL_REPLY_NOTFOUND: + ret = 0; + break; + case DICT_PROTOCOL_REPLY_FAIL: { + const char *error = strchr(line+1, '\t'); - if (ret < 0) - io_remove(&dict->io); - else if (ret > 0) { - i_error("dict-client: Unexpected reply waiting waiting for async commits: %s", line); - client_dict_disconnect(dict); + i_error("dict-client: server returned failure: %s", + error != NULL ? t_str_tabunescape(error) : ""); + break; + } + default: + ret = -1; + error = t_strdup_printf("dict-client: Invalid commit reply: %s", line); + i_error("%s", error); + client_dict_disconnect(dict, error); + break; } + dict_pre_api_callback(dict); + cmd->api_callback.commit(ret, cmd->api_callback.context); + dict_post_api_callback(dict); +} + +static void commit_sync_callback(int ret, void *context) +{ + int *ret_p = context; + *ret_p = ret; } static int @@ -758,65 +902,43 @@ client_dict_transaction_commit(struct dict_transaction_context *_ctx, struct client_dict_transaction_context *ctx = (struct client_dict_transaction_context *)_ctx; struct client_dict *dict = (struct client_dict *)_ctx->dict; - unsigned int id; - int ret = ctx->failed ? -1 : 1; + struct client_dict_cmd *cmd; + const char *query; + int ret = -1; - ctx->committed = TRUE; - if (ctx->sent_begin && !ctx->failed) T_BEGIN { - const char *query, *line; + DLLIST_REMOVE(&dict->transactions, ctx); - query = t_strdup_printf("%c%u\n", !async ? - DICT_PROTOCOL_CMD_COMMIT : - DICT_PROTOCOL_CMD_COMMIT_ASYNC, - ctx->id); - if (client_dict_send_transaction_query(ctx, query) < 0) - ret = -1; - else if (async) { - ctx->callback = callback; - ctx->context = context; - ctx->async = TRUE; - if (dict->async_commits++ == 0) { - dict->io = io_add(dict->fd, IO_READ, - dict_async_input, dict); - } + if (ctx->sent_begin && ctx->error == NULL) { + query = t_strdup_printf("%c%u", DICT_PROTOCOL_CMD_COMMIT, ctx->id); + cmd = client_dict_cmd_init(dict, query); + cmd->callback = client_dict_transaction_commit_callback; + if (callback != NULL) { + cmd->api_callback.commit = callback; + cmd->api_callback.context = context; } else { - /* sync commit, read reply */ - line = client_dict_read_line(dict); - if (line == NULL) - ret = -1; - else switch (*line) { - case DICT_PROTOCOL_REPLY_OK: - ret = 1; - break; - case DICT_PROTOCOL_REPLY_NOTFOUND: - ret = 0; - break; - case DICT_PROTOCOL_REPLY_FAIL: - ret = -1; - break; - default: - i_error("dict-client: Invalid commit reply: %s", line); - client_dict_disconnect(dict); - line = NULL; - ret = -1; - break; - } - if (line != NULL && - (str_to_uint(line+1, &id) < 0 || ctx->id != id)) { - i_error("dict-client: Invalid commit reply, " - "expected id=%u: %s", ctx->id, line); - client_dict_disconnect(dict); - ret = -1; - } + cmd->api_callback.commit = commit_sync_callback; + cmd->api_callback.context = &ret; } - } T_END; + if (client_dict_cmd_send(dict, &cmd, NULL)) { + if (!async) + client_dict_wait(_ctx->dict); + } + } else if (ctx->error != NULL) { + /* already failed */ + if (callback != NULL) + callback(-1, context); + ret = -1; + } else { + /* nothing changed */ + if (callback != NULL) + callback(1, context); + ret = 1; + } - if (ret < 0 || !async) { - DLLIST_REMOVE(&dict->transactions, ctx); - i_free(ctx); + i_free(ctx->error); + i_free(ctx); - client_dict_add_timeout(dict); - } + client_dict_add_timeout(dict); return ret; } @@ -827,13 +949,13 @@ client_dict_transaction_rollback(struct dict_transaction_context *_ctx) (struct client_dict_transaction_context *)_ctx; struct client_dict *dict = (struct client_dict *)_ctx->dict; - if (ctx->sent_begin) T_BEGIN { + if (ctx->sent_begin) { const char *query; - query = t_strdup_printf("%c%u\n", DICT_PROTOCOL_CMD_ROLLBACK, + query = t_strdup_printf("%c%u", DICT_PROTOCOL_CMD_ROLLBACK, ctx->id); client_dict_send_transaction_query(ctx, query); - } T_END; + } DLLIST_REMOVE(&dict->transactions, ctx); i_free(ctx); @@ -846,16 +968,13 @@ static void client_dict_set(struct dict_transaction_context *_ctx, { struct client_dict_transaction_context *ctx = (struct client_dict_transaction_context *)_ctx; + const char *query; - T_BEGIN { - const char *query; - - query = t_strdup_printf("%c%u\t%s\t%s\n", - DICT_PROTOCOL_CMD_SET, ctx->id, - dict_client_escape(key), - dict_client_escape(value)); - client_dict_send_transaction_query(ctx, query); - } T_END; + query = t_strdup_printf("%c%u\t%s\t%s", + DICT_PROTOCOL_CMD_SET, ctx->id, + str_tabescape(key), + str_tabescape(value)); + client_dict_send_transaction_query(ctx, query); } static void client_dict_unset(struct dict_transaction_context *_ctx, @@ -863,32 +982,12 @@ static void client_dict_unset(struct dict_transaction_context *_ctx, { struct client_dict_transaction_context *ctx = (struct client_dict_transaction_context *)_ctx; + const char *query; - T_BEGIN { - const char *query; - - query = t_strdup_printf("%c%u\t%s\n", - DICT_PROTOCOL_CMD_UNSET, ctx->id, - dict_client_escape(key)); - client_dict_send_transaction_query(ctx, query); - } T_END; -} - -static void client_dict_append(struct dict_transaction_context *_ctx, - const char *key, const char *value) -{ - struct client_dict_transaction_context *ctx = - (struct client_dict_transaction_context *)_ctx; - - T_BEGIN { - const char *query; - - query = t_strdup_printf("%c%u\t%s\t%s\n", - DICT_PROTOCOL_CMD_APPEND, ctx->id, - dict_client_escape(key), - dict_client_escape(value)); - client_dict_send_transaction_query(ctx, query); - } T_END; + query = t_strdup_printf("%c%u\t%s", + DICT_PROTOCOL_CMD_UNSET, ctx->id, + str_tabescape(key)); + client_dict_send_transaction_query(ctx, query); } static void client_dict_atomic_inc(struct dict_transaction_context *_ctx, @@ -896,34 +995,32 @@ static void client_dict_atomic_inc(struct dict_transaction_context *_ctx, { struct client_dict_transaction_context *ctx = (struct client_dict_transaction_context *)_ctx; + const char *query; - T_BEGIN { - const char *query; - query = t_strdup_printf("%c%u\t%s\t%lld\n", - DICT_PROTOCOL_CMD_ATOMIC_INC, - ctx->id, dict_client_escape(key), diff); - client_dict_send_transaction_query(ctx, query); - } T_END; + query = t_strdup_printf("%c%u\t%s\t%lld", + DICT_PROTOCOL_CMD_ATOMIC_INC, + ctx->id, str_tabescape(key), diff); + client_dict_send_transaction_query(ctx, query); } struct dict dict_driver_client = { .name = "proxy", { - client_dict_init, - client_dict_deinit, - client_dict_wait, - client_dict_lookup, - client_dict_iterate_init, - client_dict_iterate, - client_dict_iterate_deinit, - client_dict_transaction_init, - client_dict_transaction_commit, - client_dict_transaction_rollback, - client_dict_set, - client_dict_unset, - client_dict_append, - client_dict_atomic_inc, - NULL + .init = client_dict_init, + .deinit = client_dict_deinit, + .wait = client_dict_wait, + .lookup = client_dict_lookup, + .iterate_init = client_dict_iterate_init, + .iterate = client_dict_iterate, + .iterate_deinit = client_dict_iterate_deinit, + .transaction_init = client_dict_transaction_init, + .transaction_commit = client_dict_transaction_commit, + .transaction_rollback = client_dict_transaction_rollback, + .set = client_dict_set, + .unset = client_dict_unset, + .atomic_inc = client_dict_atomic_inc, + .lookup_async = client_dict_lookup_async, + .switch_ioloop = client_dict_switch_ioloop } }; diff --git a/src/lib-dict/dict-db.c b/src/lib-dict/dict-db.c index 729824b5f49..541665b2b31 100644 --- a/src/lib-dict/dict-db.c +++ b/src/lib-dict/dict-db.c @@ -465,19 +465,18 @@ db_dict_atomic_inc(struct dict_transaction_context *_ctx ATTR_UNUSED, struct dict dict_driver_db = { .name = "db", { - db_dict_init, - db_dict_deinit, - NULL, - db_dict_lookup, - db_dict_iterate_init, - db_dict_iterate, - db_dict_iterate_deinit, - db_dict_transaction_init, - db_dict_transaction_commit, - db_dict_transaction_rollback, - db_dict_set, - db_dict_unset, - db_dict_atomic_inc + .init = db_dict_init, + .deinit = db_dict_deinit, + .lookup = db_dict_lookup, + .iterate_init = db_dict_iterate_init, + .iterate = db_dict_iterate, + .iterate_deinit = db_dict_iterate_deinit, + .transaction_init = db_dict_transaction_init, + .transaction_commit = db_dict_transaction_commit, + .transaction_rollback = db_dict_transaction_rollback, + .set = db_dict_set, + .unset = db_dict_unset, + .atomic_inc = db_dict_atomic_inc, } }; #endif diff --git a/src/lib-dict/dict-file.c b/src/lib-dict/dict-file.c index b6d9f175064..ae9aa4aa01f 100644 --- a/src/lib-dict/dict-file.c +++ b/src/lib-dict/dict-file.c @@ -649,20 +649,18 @@ file_dict_transaction_commit(struct dict_transaction_context *_ctx, struct dict dict_driver_file = { .name = "file", { - file_dict_init, - file_dict_deinit, - NULL, - file_dict_lookup, - file_dict_iterate_init, - file_dict_iterate, - file_dict_iterate_deinit, - file_dict_transaction_init, - file_dict_transaction_commit, - dict_transaction_memory_rollback, - dict_transaction_memory_set, - dict_transaction_memory_unset, - dict_transaction_memory_append, - dict_transaction_memory_atomic_inc, - NULL + .init = file_dict_init, + .deinit = file_dict_deinit, + .lookup = file_dict_lookup, + .iterate_init = file_dict_iterate_init, + .iterate = file_dict_iterate, + .iterate_deinit = file_dict_iterate_deinit, + .transaction_init = file_dict_transaction_init, + .transaction_commit = file_dict_transaction_commit, + .transaction_rollback = dict_transaction_memory_rollback, + .set = dict_transaction_memory_set, + .unset = dict_transaction_memory_unset, + .append = dict_transaction_memory_append, + .atomic_inc = dict_transaction_memory_atomic_inc, } }; diff --git a/src/lib-dict/dict-memcached-ascii.c b/src/lib-dict/dict-memcached-ascii.c index 7245a0229fb..efcabd9f1af 100644 --- a/src/lib-dict/dict-memcached-ascii.c +++ b/src/lib-dict/dict-memcached-ascii.c @@ -650,20 +650,15 @@ memcached_ascii_transaction_commit(struct dict_transaction_context *_ctx, struct dict dict_driver_memcached_ascii = { .name = "memcached_ascii", { - memcached_ascii_dict_init, - memcached_ascii_dict_deinit, - NULL, - memcached_ascii_dict_lookup, - NULL, - NULL, - NULL, - memcached_ascii_transaction_init, - memcached_ascii_transaction_commit, - dict_transaction_memory_rollback, - dict_transaction_memory_set, - dict_transaction_memory_unset, - dict_transaction_memory_append, - dict_transaction_memory_atomic_inc, - NULL + .init = memcached_ascii_dict_init, + .deinit = memcached_ascii_dict_deinit, + .lookup = memcached_ascii_dict_lookup, + .transaction_init = memcached_ascii_transaction_init, + .transaction_commit = memcached_ascii_transaction_commit, + .transaction_rollback = dict_transaction_memory_rollback, + .set = dict_transaction_memory_set, + .unset = dict_transaction_memory_unset, + .append = dict_transaction_memory_append, + .atomic_inc = dict_transaction_memory_atomic_inc, } }; diff --git a/src/lib-dict/dict-memcached.c b/src/lib-dict/dict-memcached.c index e3bd6067fe4..eafd22584fe 100644 --- a/src/lib-dict/dict-memcached.c +++ b/src/lib-dict/dict-memcached.c @@ -377,20 +377,8 @@ static int memcached_dict_lookup(struct dict *_dict, pool_t pool, struct dict dict_driver_memcached = { .name = "memcached", { - memcached_dict_init, - memcached_dict_deinit, - NULL, - memcached_dict_lookup, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL, - NULL + .init = memcached_dict_init, + .deinit = memcached_dict_deinit, + .lookup = memcached_dict_lookup, } }; diff --git a/src/lib-dict/dict-private.h b/src/lib-dict/dict-private.h index 7fc0f00d2cf..f94e0552e75 100644 --- a/src/lib-dict/dict-private.h +++ b/src/lib-dict/dict-private.h @@ -38,6 +38,7 @@ struct dict_vfuncs { void (*lookup_async)(struct dict *dict, const char *key, dict_lookup_callback_t *callback, void *context); + bool (*switch_ioloop)(struct dict *dict); }; struct dict { diff --git a/src/lib-dict/dict-redis.c b/src/lib-dict/dict-redis.c index 6aa38247f1f..77e3a1dd389 100644 --- a/src/lib-dict/dict-redis.c +++ b/src/lib-dict/dict-redis.c @@ -623,8 +623,10 @@ redis_transaction_commit(struct dict_transaction_context *_ctx, bool async, redis_input_state_add(dict, REDIS_INPUT_STATE_EXEC); for (i = 0; i < ctx->cmd_count; i++) redis_input_state_add(dict, REDIS_INPUT_STATE_EXEC_REPLY); - if (async) + if (async) { + i_free(ctx); return 1; + } redis_wait(dict); } if (callback != NULL) @@ -788,20 +790,15 @@ static void redis_atomic_inc(struct dict_transaction_context *_ctx, struct dict dict_driver_redis = { .name = "redis", { - redis_dict_init, - redis_dict_deinit, - NULL, - redis_dict_lookup, - NULL, - NULL, - NULL, - redis_transaction_init, - redis_transaction_commit, - redis_transaction_rollback, - redis_set, - redis_unset, - redis_append, - redis_atomic_inc, - NULL + .init = redis_dict_init, + .deinit = redis_dict_deinit, + .lookup = redis_dict_lookup, + .transaction_init = redis_transaction_init, + .transaction_commit = redis_transaction_commit, + .transaction_rollback = redis_transaction_rollback, + .set = redis_set, + .unset = redis_unset, + .append = redis_append, + .atomic_inc = redis_atomic_inc, } }; diff --git a/src/lib-dict/dict-sql-settings.c b/src/lib-dict/dict-sql-settings.c index 8e5614633f8..25e08bbf55e 100644 --- a/src/lib-dict/dict-sql-settings.c +++ b/src/lib-dict/dict-sql-settings.c @@ -3,6 +3,7 @@ #include "lib.h" #include "array.h" #include "str.h" +#include "hash.h" #include "settings.h" #include "dict-sql-settings.h" @@ -42,6 +43,14 @@ static const struct setting_def dict_sql_map_setting_defs[] = { { 0, NULL, 0 } }; +struct dict_sql_settings_cache { + pool_t pool; + const char *path; + struct dict_sql_settings *set; +}; + +static HASH_TABLE(const char *, struct dict_sql_settings_cache *) dict_sql_settings_cache; + static const char *pattern_read_name(const char **pattern) { const char *p = *pattern, *name; @@ -235,25 +244,62 @@ parse_section(const char *type, const char *name ATTR_UNUSED, } struct dict_sql_settings * -dict_sql_settings_read(pool_t pool, const char *path, const char **error_r) +dict_sql_settings_read(const char *path, const char **error_r) { struct setting_parser_ctx ctx; + struct dict_sql_settings_cache *cache; + pool_t pool; + + if (!hash_table_is_created(dict_sql_settings_cache)) { + hash_table_create(&dict_sql_settings_cache, default_pool, 0, + str_hash, strcmp); + } + + cache = hash_table_lookup(dict_sql_settings_cache, path); + if (cache != NULL) + return cache->set; memset(&ctx, 0, sizeof(ctx)); + pool = pool_alloconly_create("dict sql settings", 1024); ctx.pool = pool; ctx.set = p_new(pool, struct dict_sql_settings, 1); t_array_init(&ctx.cur_fields, 16); p_array_init(&ctx.set->maps, pool, 8); if (!settings_read(path, NULL, parse_setting, parse_section, - &ctx, error_r)) + &ctx, error_r)) { + pool_unref(&pool); return NULL; + } if (ctx.set->connect == NULL) { *error_r = t_strdup_printf("Error in configuration file %s: " "Missing connect setting", path); + pool_unref(&pool); return NULL; } + cache = p_new(pool, struct dict_sql_settings_cache, 1); + cache->pool = pool; + cache->path = p_strdup(pool, path); + cache->set = ctx.set; + + hash_table_insert(dict_sql_settings_cache, cache->path, cache); return ctx.set; } + +void dict_sql_settings_deinit(void) +{ + struct hash_iterate_context *iter; + struct dict_sql_settings_cache *cache; + const char *key; + + if (!hash_table_is_created(dict_sql_settings_cache)) + return; + + iter = hash_table_iterate_init(dict_sql_settings_cache); + while (hash_table_iterate(iter, dict_sql_settings_cache, &key, &cache)) + pool_unref(&cache->pool); + hash_table_iterate_deinit(&iter); + hash_table_destroy(&dict_sql_settings_cache); +} diff --git a/src/lib-dict/dict-sql-settings.h b/src/lib-dict/dict-sql-settings.h index 62aff2ec17f..dd9537237bb 100644 --- a/src/lib-dict/dict-sql-settings.h +++ b/src/lib-dict/dict-sql-settings.h @@ -33,6 +33,8 @@ struct dict_sql_settings { }; struct dict_sql_settings * -dict_sql_settings_read(pool_t pool, const char *path, const char **error_r); +dict_sql_settings_read(const char *path, const char **error_r); + +void dict_sql_settings_deinit(void); #endif diff --git a/src/lib-dict/dict-sql.c b/src/lib-dict/dict-sql.c index 6af83ece99b..78df8e7dd11 100644 --- a/src/lib-dict/dict-sql.c +++ b/src/lib-dict/dict-sql.c @@ -88,7 +88,7 @@ sql_dict_init(struct dict *driver, const char *uri, dict->pool = pool; dict->dict = *driver; dict->username = p_strdup(pool, set->username); - dict->set = dict_sql_settings_read(pool, uri, error_r); + dict->set = dict_sql_settings_read(uri, error_r); if (dict->set == NULL) { pool_unref(&pool); return -1; @@ -214,6 +214,7 @@ sql_dict_find_map(struct sql_dict *dict, const char *path, static int sql_dict_value_escape(string_t *str, struct sql_dict *dict, + const struct dict_sql_map *map, enum dict_sql_type value_type, const char *field_name, const char *value, const char *value_suffix, const char **error_r) @@ -229,8 +230,8 @@ sql_dict_value_escape(string_t *str, struct sql_dict *dict, case DICT_SQL_TYPE_UINT: if (value_suffix[0] != '\0' || str_to_uint(value, &num) < 0) { *error_r = t_strdup_printf( - "field %s value isn't unsigned integer: %s%s", - field_name, value, value_suffix); + "%s field's value isn't unsigned integer: %s%s (in pattern: %s)", + field_name, value, value_suffix, map->pattern); return -1; } str_printfa(str, "%u", num); @@ -243,8 +244,8 @@ sql_dict_value_escape(string_t *str, struct sql_dict *dict, if (hex_to_binary(value, buf) < 0) { /* we shouldn't get untrusted input here. it's also a bit annoying to handle this error. */ - *error_r = t_strdup_printf("field %s value isn't hexblob: %s", - field_name, value); + *error_r = t_strdup_printf("%s field's value isn't hexblob: %s (in pattern: %s)", + field_name, value, map->pattern); return -1; } str_append(buf, value_suffix); @@ -254,11 +255,12 @@ sql_dict_value_escape(string_t *str, struct sql_dict *dict, static int sql_dict_field_escape_value(string_t *str, struct sql_dict *dict, + const struct dict_sql_map *map, const struct dict_sql_field *field, const char *value, const char *value_suffix, const char **error_r) { - return sql_dict_value_escape(str, dict, field->value_type, + return sql_dict_value_escape(str, dict, map, field->value_type, field->name, value, value_suffix, error_r); } @@ -290,7 +292,7 @@ sql_dict_where_build(struct sql_dict *dict, const struct dict_sql_map *map, if (i > 0) str_append(query, " AND"); str_printfa(query, " %s = ", sql_fields[i].name); - if (sql_dict_field_escape_value(query, dict, &sql_fields[i], + if (sql_dict_field_escape_value(query, dict, map, &sql_fields[i], values[i], "", error_r) < 0) return -1; } @@ -302,11 +304,11 @@ sql_dict_where_build(struct sql_dict *dict, const struct dict_sql_map *map, str_append(query, " AND"); if (i < count2) { str_printfa(query, " %s LIKE ", sql_fields[i].name); - if (sql_dict_field_escape_value(query, dict, &sql_fields[i], + if (sql_dict_field_escape_value(query, dict, map, &sql_fields[i], values[i], "/%", error_r) < 0) return -1; str_printfa(query, " AND %s NOT LIKE ", sql_fields[i].name); - if (sql_dict_field_escape_value(query, dict, &sql_fields[i], + if (sql_dict_field_escape_value(query, dict, map, &sql_fields[i], values[i], "/%/%", error_r) < 0) return -1; } else { @@ -321,7 +323,7 @@ sql_dict_where_build(struct sql_dict *dict, const struct dict_sql_map *map, str_append(query, " AND"); str_printfa(query, " %s LIKE ", sql_fields[i].name); - if (sql_dict_field_escape_value(query, dict, &sql_fields[i], + if (sql_dict_field_escape_value(query, dict, map, &sql_fields[i], values[i], "/%", error_r) < 0) return -1; } @@ -476,6 +478,12 @@ sql_dict_lookup_async_callback(struct sql_result *sql_result, else if (result.ret > 0) { result.value = sql_dict_result_unescape_value(ctx->map, pool_datastack_create(), sql_result); + if (result.value == NULL) { + /* NULL value returned. we'll treat this as + "not found", which is probably what is usually + wanted. */ + result.ret = 0; + } } ctx->callback(&result, ctx->context); @@ -908,8 +916,9 @@ static int sql_dict_set_query(const struct dict_sql_build_query *build, else { enum dict_sql_type value_type = sql_dict_map_type(fields[i].map); - if (sql_dict_value_escape(suffix, dict, value_type, - "value", fields[i].value, "", error_r) < 0) + if (sql_dict_value_escape(suffix, dict, fields[i].map, + value_type, "value", fields[i].value, + "", error_r) < 0) return -1; } } @@ -926,7 +935,7 @@ static int sql_dict_set_query(const struct dict_sql_build_query *build, for (i = 0; i < count; i++) { str_printfa(prefix, ",%s", sql_fields[i].name); str_append_c(suffix, ','); - if (sql_dict_field_escape_value(suffix, dict, &sql_fields[i], + if (sql_dict_field_escape_value(suffix, dict, fields[0].map, &sql_fields[i], extra_values[i], "", error_r) < 0) return -1; } @@ -951,8 +960,9 @@ static int sql_dict_set_query(const struct dict_sql_build_query *build, } else { enum dict_sql_type value_type = sql_dict_map_type(fields[i].map); - if (sql_dict_value_escape(prefix, dict, value_type, - "value", fields[i].value, "", error_r) < 0) + if (sql_dict_value_escape(prefix, dict, fields[i].map, + value_type, "value", fields[i].value, + "", error_r) < 0) return -1; } } @@ -1242,21 +1252,21 @@ static struct dict sql_dict = { .name = "sql", { - sql_dict_init, - sql_dict_deinit, - sql_dict_wait, - sql_dict_lookup, - sql_dict_iterate_init, - sql_dict_iterate, - sql_dict_iterate_deinit, - sql_dict_transaction_init, - sql_dict_transaction_commit, - sql_dict_transaction_rollback, - sql_dict_set, - sql_dict_unset, - sql_dict_append, - sql_dict_atomic_inc, - sql_dict_lookup_async + .init = sql_dict_init, + .deinit = sql_dict_deinit, + .wait = sql_dict_wait, + .lookup = sql_dict_lookup, + .iterate_init = sql_dict_iterate_init, + .iterate = sql_dict_iterate, + .iterate_deinit = sql_dict_iterate_deinit, + .transaction_init = sql_dict_transaction_init, + .transaction_commit = sql_dict_transaction_commit, + .transaction_rollback = sql_dict_transaction_rollback, + .set = sql_dict_set, + .unset = sql_dict_unset, + .append = sql_dict_append, + .atomic_inc = sql_dict_atomic_inc, + .lookup_async = sql_dict_lookup_async, } }; @@ -1289,4 +1299,5 @@ void dict_sql_unregister(void) dict_driver_unregister(&dict_sql_drivers[i]); i_free(dict_sql_drivers); sql_db_cache_deinit(&dict_sql_db_cache); + dict_sql_settings_deinit(); } diff --git a/src/lib-dict/dict.c b/src/lib-dict/dict.c index e3d1deb2f5d..f6d3f68eead 100644 --- a/src/lib-dict/dict.c +++ b/src/lib-dict/dict.c @@ -106,6 +106,14 @@ int dict_wait(struct dict *dict) return dict->v.wait == NULL ? 1 : dict->v.wait(dict); } +bool dict_switch_ioloop(struct dict *dict) +{ + if (dict->v.switch_ioloop != NULL) + return dict->v.switch_ioloop(dict); + else + return FALSE; +} + static bool dict_key_prefix_is_valid(const char *key) { return strncmp(key, DICT_PATH_SHARED, strlen(DICT_PATH_SHARED)) == 0 || diff --git a/src/lib-dict/dict.h b/src/lib-dict/dict.h index ed661a9ac19..6188a1a1b71 100644 --- a/src/lib-dict/dict.h +++ b/src/lib-dict/dict.h @@ -70,6 +70,10 @@ void dict_deinit(struct dict **dict); /* Wait for all pending asynchronous operations to finish. Returns 0 if ok, -1 if error. */ int dict_wait(struct dict *dict); +/* Switch the dict to the current ioloop. This can be used to do dict_wait() + among other IO work. Returns TRUE if there is actually some work that can + be waited on. */ +bool dict_switch_ioloop(struct dict *dict) ATTR_NOWARN_UNUSED_RESULT; /* Lookup value for key. Set it to NULL if it's not found. Returns 1 if found, 0 if not found and -1 if lookup failed. */ diff --git a/src/lib-dovecot/Makefile.am b/src/lib-dovecot/Makefile.am index fd087760869..a8820e5e197 100644 --- a/src/lib-dovecot/Makefile.am +++ b/src/lib-dovecot/Makefile.am @@ -1,28 +1,10 @@ -# when adding libraries, update LIBDOVECOT also in configure.in -libs = \ - ../lib-master/libmaster.la \ - ../lib-fs/libfs.la \ - ../lib-settings/libsettings.la \ - ../lib-stats/libstats.la \ - ../lib-http/libhttp.la \ - ../lib-dict/libdict.la \ - ../lib-imap/libimap.la \ - ../lib-mail/libmail.la \ - ../lib-sasl/libsasl.la \ - ../lib-auth/libauth.la \ - ../lib-dns/libdns.la \ - ../lib-charset/libcharset.la \ - ../lib-ssl-iostream/libssl_iostream.la \ - ../lib-test/libtest.la \ - ../lib/liblib.la - pkglib_LTLIBRARIES = libdovecot.la libdovecot_la_SOURCES = libdovecot_la_LIBADD = \ - $(libs) \ + $(LIBDOVECOT_LA_LIBS) \ $(MODULE_LIBS) -libdovecot_la_DEPENDENCIES = $(libs) +libdovecot_la_DEPENDENCIES = $(LIBDOVECOT_LA_LIBS) libdovecot_la_LDFLAGS = -export-dynamic diff --git a/src/lib-fs/Makefile.am b/src/lib-fs/Makefile.am index aed58c18f37..96930462eaa 100644 --- a/src/lib-fs/Makefile.am +++ b/src/lib-fs/Makefile.am @@ -2,6 +2,7 @@ noinst_LTLIBRARIES = libfs.la AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ -I$(top_srcdir)/src/lib-dict \ -I$(top_srcdir)/src/lib-ssl-iostream \ -DMODULE_DIR=\""$(moduledir)"\" @@ -12,6 +13,8 @@ libfs_la_SOURCES = \ fs-metawrap.c \ fs-randomfail.c \ fs-posix.c \ + fs-test.c \ + fs-test-async.c \ fs-sis.c \ fs-sis-common.c \ fs-sis-queue.c \ @@ -25,6 +28,7 @@ headers = \ fs-api.h \ fs-api-private.h \ fs-sis-common.h \ + fs-test.h \ istream-fs-file.h \ istream-fs-stats.h \ istream-metawrap.h \ @@ -33,3 +37,28 @@ headers = \ pkginc_libdir=$(pkgincludedir) pkginc_lib_HEADERS = $(headers) + +noinst_PROGRAMS = $(test_programs) + +test_programs = \ + test-fs-metawrap + +test_deps = \ + $(noinst_LTLIBRARIES) \ + ../lib-dict/libdict.la \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +test_libs = \ + $(test_deps) \ + $(MODULE_LIBS) + +test_fs_metawrap_SOURCES = test-fs-metawrap.c +test_fs_metawrap_LDADD = $(test_libs) +test_fs_metawrap_DEPENDENCIES = $(test_deps) + +check: check-am check-test +check-test: all-am + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done diff --git a/src/lib-fs/fs-api-private.h b/src/lib-fs/fs-api-private.h index 8e6d173f5eb..fa5168b575a 100644 --- a/src/lib-fs/fs-api-private.h +++ b/src/lib-fs/fs-api-private.h @@ -48,6 +48,8 @@ struct fs_vfuncs { int (*write)(struct fs_file *file, const void *data, size_t size); void (*write_stream)(struct fs_file *file); + /* After write_stream_finish() is called once, all the following + (async) calls will have success==TRUE. */ int (*write_stream_finish)(struct fs_file *file, bool success); int (*lock)(struct fs_file *file, unsigned int secs, @@ -64,6 +66,8 @@ struct fs_vfuncs { enum fs_iter_flags flags); const char *(*iter_next)(struct fs_iter *iter); int (*iter_deinit)(struct fs_iter *iter); + + bool (*switch_ioloop)(struct fs *fs); }; struct fs { @@ -115,6 +119,7 @@ struct fs_file { struct timeval timing_start[FS_OP_COUNT]; unsigned int write_pending:1; + unsigned int writing_stream:1; unsigned int metadata_changed:1; unsigned int read_or_prefetch_counted:1; @@ -145,6 +150,7 @@ extern const struct fs fs_class_randomfail; extern const struct fs fs_class_metawrap; extern const struct fs fs_class_sis; extern const struct fs fs_class_sis_queue; +extern const struct fs fs_class_test; void fs_class_register(const struct fs *fs_class); diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c index 921485d47ae..7bd981eb3b0 100644 --- a/src/lib-fs/fs-api.c +++ b/src/lib-fs/fs-api.c @@ -74,6 +74,7 @@ static void fs_classes_init(void) fs_class_register(&fs_class_metawrap); fs_class_register(&fs_class_sis); fs_class_register(&fs_class_sis_queue); + fs_class_register(&fs_class_test); lib_atexit(fs_classes_deinit); } @@ -261,6 +262,9 @@ void fs_file_deinit(struct fs_file **_file) void fs_file_close(struct fs_file *file) { + i_assert(!file->writing_stream); + i_assert(file->output == NULL); + if (file->pending_read_input != NULL) i_stream_unref(&file->pending_read_input); if (file->seekable_input != NULL) @@ -508,9 +512,10 @@ struct istream *fs_read_stream(struct fs_file *file, size_t max_buffer_size) } if (file->seekable_input != NULL) { - i_stream_seek(file->seekable_input, 0); - i_stream_ref(file->seekable_input); - return file->seekable_input; + /* allow multiple open streams, each in a different position */ + input = i_stream_create_limit(file->seekable_input, (uoff_t)-1); + i_stream_seek(input, 0); + return input; } T_BEGIN { input = file->fs->v.read_stream(file, max_buffer_size); @@ -540,10 +545,10 @@ struct istream *fs_read_stream(struct fs_file *file, size_t max_buffer_size) file->fs->temp_path_prefix); i_stream_set_name(input, i_stream_get_name(inputs[0])); i_stream_unref(&inputs[0]); - - file->seekable_input = input; - i_stream_ref(file->seekable_input); } + file->seekable_input = input; + i_stream_ref(file->seekable_input); + if ((file->flags & FS_OPEN_FLAG_ASYNC) == 0 && !input->blocking) { /* read the whole input stream before returning */ while ((ret = i_stream_read_data(input, &data, &size, 0)) >= 0) { @@ -603,6 +608,7 @@ int fs_write(struct fs_file *file, const void *data, size_t size) } T_END; if (!(ret < 0 && errno == EAGAIN)) { file->fs->stats.write_count++; + file->fs->stats.write_bytes += size; fs_file_timing_end(file, FS_OP_WRITE); } return ret; @@ -615,6 +621,10 @@ int fs_write(struct fs_file *file, const void *data, size_t size) struct ostream *fs_write_stream(struct fs_file *file) { + i_assert(!file->writing_stream); + i_assert(file->output == NULL); + + file->writing_stream = TRUE; file->fs->stats.write_count++; T_BEGIN { file->fs->v.write_stream(file); @@ -628,6 +638,8 @@ static int fs_write_stream_finish_int(struct fs_file *file, bool success) { int ret; + i_assert(file->writing_stream); + fs_file_timing_start(file, FS_OP_WRITE); T_BEGIN { ret = file->fs->v.write_stream_finish(file, success); @@ -640,6 +652,10 @@ static int fs_write_stream_finish_int(struct fs_file *file, bool success) indicated a failure. */ i_assert(success); } + if (ret != 0) { + i_assert(file->output == NULL); + file->writing_stream = FALSE; + } return ret; } @@ -648,17 +664,18 @@ int fs_write_stream_finish(struct fs_file *file, struct ostream **output) bool success = TRUE; i_assert(*output == file->output || *output == NULL); + i_assert(output != &file->output); *output = NULL; - if (file->output != NULL) - o_stream_uncork(file->output); if (file->output != NULL) { + o_stream_uncork(file->output); if (o_stream_nfinish(file->output) < 0) { fs_set_error(file->fs, "write(%s) failed: %s", o_stream_get_name(file->output), o_stream_get_error(file->output)); success = FALSE; } + file->fs->stats.write_bytes += file->output->offset; } return fs_write_stream_finish_int(file, success); } @@ -670,12 +687,18 @@ int fs_write_stream_finish_async(struct fs_file *file) void fs_write_stream_abort(struct fs_file *file, struct ostream **output) { + int ret; + i_assert(*output == file->output); + i_assert(file->output != NULL); + i_assert(output != &file->output); *output = NULL; - if (file->output != NULL) - o_stream_ignore_last_errors(file->output); - (void)fs_write_stream_finish_int(file, FALSE); + o_stream_ignore_last_errors(file->output); + /* make sure we don't have an old error lying around */ + fs_set_error(file->fs, "Write aborted"); + ret = fs_write_stream_finish_int(file, FALSE); + i_assert(ret != 0); } void fs_write_set_hash(struct fs_file *file, const struct hash_method *method, @@ -716,6 +739,20 @@ int fs_wait_async(struct fs *fs) return ret; } +bool fs_switch_ioloop(struct fs *fs) +{ + bool ret = FALSE; + + if (fs->v.switch_ioloop != NULL) { + T_BEGIN { + ret = fs->v.switch_ioloop(fs); + } T_END; + } else if (fs->parent != NULL) { + ret = fs_switch_ioloop(fs->parent); + } + return ret; +} + int fs_lock(struct fs_file *file, unsigned int secs, struct fs_lock **lock_r) { int ret; @@ -748,13 +785,14 @@ int fs_exists(struct fs_file *file) else return errno == ENOENT ? 0 : -1; } - file->fs->stats.exists_count++; fs_file_timing_start(file, FS_OP_EXISTS); T_BEGIN { ret = file->fs->v.exists(file); } T_END; - if (!(ret < 0 && errno == EAGAIN)) + if (!(ret < 0 && errno == EAGAIN)) { + file->fs->stats.exists_count++; fs_file_timing_end(file, FS_OP_EXISTS); + } return ret; } @@ -842,13 +880,13 @@ int fs_copy(struct fs_file *src, struct fs_file *dest) return -1; } - dest->fs->stats.copy_count++; fs_file_timing_start(dest, FS_OP_COPY); T_BEGIN { ret = src->fs->v.copy(src, dest); } T_END; if (!(ret < 0 && errno == EAGAIN)) { fs_file_timing_end(dest, FS_OP_COPY); + dest->fs->stats.copy_count++; dest->metadata_changed = FALSE; } return ret; @@ -863,6 +901,7 @@ int fs_copy_finish_async(struct fs_file *dest) } T_END; if (!(ret < 0 && errno == EAGAIN)) { fs_file_timing_end(dest, FS_OP_COPY); + dest->fs->stats.copy_count++; dest->metadata_changed = FALSE; } return ret; @@ -874,13 +913,14 @@ int fs_rename(struct fs_file *src, struct fs_file *dest) i_assert(src->fs == dest->fs); - dest->fs->stats.rename_count++; fs_file_timing_start(dest, FS_OP_RENAME); T_BEGIN { ret = src->fs->v.rename(src, dest); } T_END; - if (!(ret < 0 && errno == EAGAIN)) + if (!(ret < 0 && errno == EAGAIN)) { + dest->fs->stats.rename_count++; fs_file_timing_end(dest, FS_OP_RENAME); + } return ret; } @@ -888,13 +928,16 @@ int fs_delete(struct fs_file *file) { int ret; - file->fs->stats.delete_count++; + i_assert(!file->writing_stream); + fs_file_timing_start(file, FS_OP_DELETE); T_BEGIN { ret = file->fs->v.delete_file(file); } T_END; - if (!(ret < 0 && errno == EAGAIN)) + if (!(ret < 0 && errno == EAGAIN)) { + file->fs->stats.delete_count++; fs_file_timing_end(file, FS_OP_DELETE); + } return ret; } diff --git a/src/lib-fs/fs-api.h b/src/lib-fs/fs-api.h index c21080386c2..21979d4b14c 100644 --- a/src/lib-fs/fs-api.h +++ b/src/lib-fs/fs-api.h @@ -36,7 +36,9 @@ enum fs_properties { /* Backend support asynchronous file operations. */ FS_PROPERTY_ASYNC = 0x800, /* Backend supports FS_ITER_FLAG_OBJECTIDS. */ - FS_PROPERTY_OBJECTIDS = 0x1000 + FS_PROPERTY_OBJECTIDS = 0x1000, + /* fs_copy() is fast even when file's metadata is changed */ + FS_PROPERTY_FASTCOPY_CHANGED_METADATA = 0x2000, }; enum fs_open_mode { @@ -173,6 +175,9 @@ struct fs_stats { /* Number of fs_iter_init() calls. */ unsigned int iter_count; + /* Number of bytes written by fs_write*() calls. */ + uint64_t write_bytes; + /* Cumulative sum of usecs spent on calls - set only if fs_settings.enable_timing=TRUE */ struct timing *timings[FS_OP_COUNT]; @@ -266,7 +271,9 @@ int fs_write_stream_finish(struct fs_file *file, struct ostream **output); int fs_write_stream_finish_async(struct fs_file *file); /* Abort writing via stream. Anything written to the stream is discarded. o_stream_ignore_last_errors() is called on the output stream so the caller - doesn't need to do it. */ + doesn't need to do it. This must not be called after + fs_write_stream_finish(), i.e. it can't be used to abort a pending async + write. */ void fs_write_stream_abort(struct fs_file *file, struct ostream **output); /* Set a hash to the following write. The storage can then verify that the @@ -284,6 +291,10 @@ void fs_file_set_async_callback(struct fs_file *file, It's an error to call this when there are no pending async operations. Returns 0 if ok, -1 if timed out. */ int fs_wait_async(struct fs *fs); +/* Switch the fs to the current ioloop. This can be used to do fs_wait_async() + among other IO work. Returns TRUE if there is actually some work that can + be waited on. */ +bool fs_switch_ioloop(struct fs *fs) ATTR_NOWARN_UNUSED_RESULT; /* Returns 1 if file exists, 0 if not, -1 if error occurred. */ int fs_exists(struct fs_file *file); diff --git a/src/lib-fs/fs-dict.c b/src/lib-fs/fs-dict.c index 27049502988..5efd33abb65 100644 --- a/src/lib-fs/fs-dict.c +++ b/src/lib-fs/fs-dict.c @@ -318,6 +318,7 @@ const struct fs fs_class_dict = { fs_dict_delete, fs_dict_iter_init, fs_dict_iter_next, - fs_dict_iter_deinit + fs_dict_iter_deinit, + NULL } }; diff --git a/src/lib-fs/fs-metawrap.c b/src/lib-fs/fs-metawrap.c index a8c9f8ace52..85b2f059a03 100644 --- a/src/lib-fs/fs-metawrap.c +++ b/src/lib-fs/fs-metawrap.c @@ -13,8 +13,6 @@ #include "iostream-temp.h" #include "fs-api-private.h" -#define MAX_METADATA_LINE_LEN 8192 - struct metawrap_fs { struct fs fs; bool wrap_metadata; @@ -70,7 +68,7 @@ fs_metawrap_init(struct fs *_fs, const char *args, const parent_args++; } if (fs_init(parent_name, parent_args, set, &_fs->parent, &error) < 0) { - fs_set_error(_fs, "%s: %s", parent_name, error); + fs_set_error(_fs, "%s", error); return -1; } if ((fs_get_properties(_fs->parent) & FS_PROPERTY_METADATA) == 0) @@ -199,14 +197,34 @@ fs_metawrap_get_metadata(struct fs_file *_file, const ARRAY_TYPE(fs_metadata) **metadata_r) { struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file; + ssize_t ret; char c; if (!file->fs->wrap_metadata) return fs_get_metadata(file->super, metadata_r); - if (!file->metadata_read) { + if (file->metadata_read) { + /* we have the metadata */ + } else if (file->input == NULL) { if (fs_read(_file, &c, 1) < 0) return -1; + } else { + /* use the existing istream to read it */ + while ((ret = i_stream_read(file->input)) == 0) { + if (file->metadata_read) + break; + + i_assert(!file->input->blocking); + if (fs_wait_async(_file->fs) < 0) + return -1; + } + if (ret == -1 && file->input->stream_errno != 0) { + fs_set_error(_file->fs, "read(%s) failed: %s", + i_stream_get_name(file->input), + i_stream_get_error(file->input)); + return -1; + } + i_assert(file->metadata_read); } *metadata_r = &_file->metadata; return 0; @@ -263,8 +281,7 @@ fs_metawrap_read_stream(struct fs_file *_file, size_t max_buffer_size) return file->input; } - input = fs_read_stream(file->super_read, - I_MAX(max_buffer_size, MAX_METADATA_LINE_LEN)); + input = fs_read_stream(file->super_read, max_buffer_size); file->input = i_stream_create_metawrap(input, fs_metawrap_callback, file); i_stream_unref(&input); i_stream_ref(file->input); @@ -366,18 +383,20 @@ static int fs_metawrap_write_stream_finish(struct fs_file *_file, bool success) int ret; if (_file->output != NULL) { - if (_file->output->closed) - success = FALSE; if (_file->output == file->super_output) _file->output = NULL; else o_stream_unref(&_file->output); } if (!success) { - if (file->temp_output != NULL) - o_stream_destroy(&file->temp_output); - if (file->super_output != NULL) + if (file->super_output != NULL) { + /* no metawrap */ + i_assert(file->temp_output == NULL); fs_write_stream_abort(file->super, &file->super_output); + } else { + i_assert(file->temp_output != NULL); + o_stream_destroy(&file->temp_output); + } return -1; } @@ -389,7 +408,7 @@ static int fs_metawrap_write_stream_finish(struct fs_file *_file, bool success) if (file->temp_output == NULL) { /* finishing up */ i_assert(file->super_output == NULL); - return fs_write_stream_finish(file->super, &file->temp_output); + return fs_write_stream_finish_async(file->super); } /* finish writing the temporary file */ input = iostream_temp_finish(&file->temp_output, IO_BLOCK_SIZE); @@ -403,21 +422,22 @@ static int fs_metawrap_write_stream_finish(struct fs_file *_file, bool success) i_stream_unref(&input2); } file->super_output = fs_write_stream(file->super); - if (o_stream_send_istream(file->super_output, input) >= 0) - ret = fs_write_stream_finish(file->super, &file->super_output); - else if (input->stream_errno != 0) { + (void)o_stream_send_istream(file->super_output, input); + if (input->stream_errno != 0) { fs_set_error(_file->fs, "read(%s) failed: %s", i_stream_get_name(input), i_stream_get_error(input)); fs_write_stream_abort(file->super, &file->super_output); ret = -1; - } else { - i_assert(file->super_output->stream_errno != 0); + } else if (file->super_output->stream_errno != 0) { fs_set_error(_file->fs, "write(%s) failed: %s", o_stream_get_name(file->super_output), o_stream_get_error(file->super_output)); fs_write_stream_abort(file->super, &file->super_output); ret = -1; + } else { + i_assert(i_stream_is_eof(input)); + ret = fs_write_stream_finish(file->super, &file->super_output); } i_stream_unref(&input); return ret; @@ -469,7 +489,12 @@ static int fs_metawrap_stat(struct fs_file *_file, struct stat *st_r) return 0; } - input = fs_read_stream(_file, IO_BLOCK_SIZE); + if (file->input == NULL) + input = fs_read_stream(_file, IO_BLOCK_SIZE); + else { + input = file->input; + i_stream_ref(input); + } if ((ret = i_stream_get_size(input, TRUE, &input_size)) < 0) { fs_set_error(_file->fs, "i_stream_get_size(%s) failed: %s", fs_file_path(_file), i_stream_get_error(input)); @@ -478,7 +503,10 @@ static int fs_metawrap_stat(struct fs_file *_file, struct stat *st_r) } i_stream_unref(&input); if (ret == 0) { - fs_set_error_async(_file->fs); + /* we shouldn't get here */ + fs_set_error(_file->fs, "i_stream_get_size(%s) returned size as unknown", + fs_file_path(_file)); + errno = EIO; return -1; } @@ -585,6 +613,7 @@ const struct fs fs_class_metawrap = { fs_metawrap_delete, fs_metawrap_iter_init, fs_metawrap_iter_next, - fs_metawrap_iter_deinit + fs_metawrap_iter_deinit, + NULL } }; diff --git a/src/lib-fs/fs-posix.c b/src/lib-fs/fs-posix.c index 3e4a7792375..3acad9347fb 100644 --- a/src/lib-fs/fs-posix.c +++ b/src/lib-fs/fs-posix.c @@ -35,6 +35,7 @@ struct posix_fs { enum fs_posix_lock_method lock_method; mode_t mode; bool mode_auto; + bool have_dirs; }; struct posix_fs_file { @@ -47,7 +48,6 @@ struct posix_fs_file { buffer_t *write_buf; bool seek_to_beginning; - bool success; }; struct posix_fs_lock { @@ -88,7 +88,7 @@ fs_posix_init(struct fs *_fs, const char *args, const struct fs_settings *set) fs->lock_method = FS_POSIX_LOCK_METHOD_FLOCK; fs->mode = FS_DEFAULT_MODE; - tmp = t_strsplit_spaces(args, " "); + tmp = t_strsplit_spaces(args, ": "); for (; *tmp != NULL; tmp++) { const char *arg = *tmp; @@ -105,6 +105,8 @@ fs_posix_init(struct fs *_fs, const char *args, const struct fs_settings *set) fs->path_prefix = i_strdup(arg + 7); } else if (strcmp(arg, "mode=auto") == 0) { fs->mode_auto = TRUE; + } else if (strcmp(arg, "dirs") == 0) { + fs->have_dirs = TRUE; } else if (strncmp(arg, "mode=", 5) == 0) { unsigned int mode; if (str_to_uint_oct(arg+5, &mode) < 0) { @@ -134,13 +136,21 @@ static void fs_posix_deinit(struct fs *_fs) i_free(fs); } -static enum fs_properties fs_posix_get_properties(struct fs *fs ATTR_UNUSED) +static enum fs_properties fs_posix_get_properties(struct fs *_fs) { - /* FS_PROPERTY_DIRECTORIES not returned because fs_delete() - automatically rmdir()s parents. This could be changed later though, - but SIS code at least would need to be changed to support it. */ - return FS_PROPERTY_LOCKS | FS_PROPERTY_FASTCOPY | FS_PROPERTY_RENAME | + struct posix_fs *fs = (struct posix_fs *)_fs; + enum fs_properties props = + FS_PROPERTY_LOCKS | FS_PROPERTY_FASTCOPY | FS_PROPERTY_RENAME | FS_PROPERTY_STAT | FS_PROPERTY_ITER | FS_PROPERTY_RELIABLEITER; + + /* FS_PROPERTY_DIRECTORIES is not returned normally because fs_delete() + automatically rmdir()s parents. For backwards compatibility + (especially with SIS code) we'll do it that way, but optionally with + "dirs" parameter enable them. This is especially important to be + able to use doveadm fs commands to delete empty directories. */ + if (fs->have_dirs) + props |= FS_PROPERTY_DIRECTORIES; + return props; } static int @@ -202,12 +212,15 @@ static int fs_posix_rmdir_parents(struct posix_fs *fs, const char *path) { const char *p; - if (fs->root_path == NULL) + if (fs->have_dirs) + return 0; + if (fs->root_path == NULL && fs->path_prefix == NULL) return 0; while ((p = strrchr(path, '/')) != NULL) { path = t_strdup_until(path, p); - if (strcmp(path, fs->root_path) == 0) + if ((fs->root_path != NULL && strcmp(path, fs->root_path) == 0) || + (fs->path_prefix != NULL && strncmp(path, fs->path_prefix, strlen(path)) == 0)) break; if (rmdir(path) == 0) { /* success, continue to parent */ @@ -345,7 +358,7 @@ static void fs_posix_file_deinit(struct fs_file *_file) case FS_OPEN_MODE_CREATE_UNIQUE_128: case FS_OPEN_MODE_CREATE: case FS_OPEN_MODE_REPLACE: - if (file->success || file->temp_path == NULL) + if (file->temp_path == NULL) break; /* failed to create/replace this. delete the temp file */ if (unlink(file->temp_path) < 0) { @@ -432,7 +445,7 @@ fs_posix_read_stream(struct fs_file *_file, size_t max_buffer_size) static int fs_posix_write_finish(struct posix_fs_file *file) { - int ret; + int ret, old_errno; if ((file->open_flags & FS_OPEN_FLAG_FSYNC) != 0) { if (fdatasync(file->fd) < 0) { @@ -449,10 +462,12 @@ static int fs_posix_write_finish(struct posix_fs_file *file) fs_set_error(file->file.fs, "link(%s, %s) failed: %m", file->temp_path, file->full_path); } + old_errno = errno; if (unlink(file->temp_path) < 0) { fs_set_error(file->file.fs, "unlink(%s) failed: %m", file->temp_path); } + errno = old_errno; if (ret < 0) { fs_posix_file_close(&file->file); i_free_and_null(file->temp_path); @@ -470,7 +485,6 @@ static int fs_posix_write_finish(struct posix_fs_file *file) i_unreached(); } i_free_and_null(file->temp_path); - file->success = TRUE; file->seek_to_beginning = TRUE; /* allow opening the file after writing to it */ file->open_mode = FS_OPEN_MODE_READONLY; @@ -873,6 +887,7 @@ const struct fs fs_class_posix = { fs_posix_delete, fs_posix_iter_init, fs_posix_iter_next, - fs_posix_iter_deinit + fs_posix_iter_deinit, + NULL } }; diff --git a/src/lib-fs/fs-randomfail.c b/src/lib-fs/fs-randomfail.c index 62999204b1d..ff4e80f76ee 100644 --- a/src/lib-fs/fs-randomfail.c +++ b/src/lib-fs/fs-randomfail.c @@ -27,6 +27,7 @@ struct randomfail_fs_file { struct fs_file file; struct fs_file *super, *super_read; struct istream *input; + bool op_pending[FS_OP_COUNT]; struct ostream *super_output; }; @@ -188,7 +189,7 @@ fs_randomfail_init(struct fs *_fs, const char *args, parent_args++; } if (fs_init(parent_name, parent_args, set, &_fs->parent, &error) < 0) { - fs_set_error(_fs, "%s: %s", parent_name, error); + fs_set_error(_fs, "%s", error); return -1; } return 0; @@ -254,13 +255,40 @@ fs_randomfail_set_async_callback(struct fs_file *_file, fs_file_set_async_callback(file->super, callback, context); } -static bool fs_random_fail(struct fs *_fs, enum fs_op op) +static bool fs_random_fail(struct fs *_fs, int divider, enum fs_op op) { struct randomfail_fs *fs = (struct randomfail_fs *)_fs; if (fs->op_probability[op] == 0) return FALSE; - return (unsigned int)(rand() % 100) <= fs->op_probability[op]; + if ((unsigned int)(rand() % (100*divider)) <= fs->op_probability[op]) { + fs_set_error(_fs, RANDOMFAIL_ERROR); + return TRUE; + } + return FALSE; +} + +static bool +fs_file_random_fail_begin(struct randomfail_fs_file *file, enum fs_op op) +{ + if (!file->op_pending[op]) { + if (fs_random_fail(file->file.fs, 2, op)) + return TRUE; + } + file->op_pending[op] = TRUE; + return FALSE; +} + +static int +fs_file_random_fail_end(struct randomfail_fs_file *file, + int ret, enum fs_op op) +{ + if (ret == 0 || errno != ENOENT) { + if (fs_random_fail(file->file.fs, 2, op)) + return TRUE; + file->op_pending[op] = FALSE; + } + return ret; } static bool @@ -268,7 +296,7 @@ fs_random_fail_range(struct fs *_fs, enum fs_op op, uoff_t *offset_r) { struct randomfail_fs *fs = (struct randomfail_fs *)_fs; - if (!fs_random_fail(_fs, op)) + if (!fs_random_fail(_fs, 1, op)) return FALSE; *offset_r = fs->range_start[op] + rand() % (fs->range_end[op] - fs->range_start[op] + 1); @@ -277,7 +305,7 @@ fs_random_fail_range(struct fs *_fs, enum fs_op op, uoff_t *offset_r) static int fs_randomfail_wait_async(struct fs *_fs) { - if (fs_random_fail(_fs, FS_OP_WAIT)) + if (fs_random_fail(_fs, 1, FS_OP_WAIT)) return -1; return fs_wait_async(_fs->parent); } @@ -296,17 +324,19 @@ fs_randomfail_get_metadata(struct fs_file *_file, const ARRAY_TYPE(fs_metadata) **metadata_r) { struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file; + int ret; - if (fs_random_fail(_file->fs, FS_OP_METADATA)) + if (fs_file_random_fail_begin(file, FS_OP_METADATA)) return -1; - return fs_get_metadata(file->super, metadata_r); + ret = fs_get_metadata(file->super, metadata_r); + return fs_file_random_fail_end(file, ret, FS_OP_METADATA); } static bool fs_randomfail_prefetch(struct fs_file *_file, uoff_t length) { struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file; - if (fs_random_fail(_file->fs, FS_OP_PREFETCH)) + if (fs_random_fail(_file->fs, 1, FS_OP_PREFETCH)) return TRUE; return fs_prefetch(file->super, length); } @@ -314,10 +344,12 @@ static bool fs_randomfail_prefetch(struct fs_file *_file, uoff_t length) static ssize_t fs_randomfail_read(struct fs_file *_file, void *buf, size_t size) { struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file; + int ret; - if (fs_random_fail(_file->fs, FS_OP_READ)) + if (fs_file_random_fail_begin(file, FS_OP_READ)) return -1; - return fs_read(file->super, buf, size); + ret = fs_read(file->super, buf, size); + return fs_file_random_fail_end(file, ret, FS_OP_READ); } static struct istream * @@ -338,10 +370,12 @@ fs_randomfail_read_stream(struct fs_file *_file, size_t max_buffer_size) static int fs_randomfail_write(struct fs_file *_file, const void *data, size_t size) { struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file; + int ret; - if (fs_random_fail(_file->fs, FS_OP_WRITE)) + if (fs_file_random_fail_begin(file, FS_OP_WRITE)) return -1; - return fs_write(file->super, data, size); + ret = fs_write(file->super, data, size); + return fs_file_random_fail_end(file, ret, FS_OP_EXISTS); } static void fs_randomfail_write_stream(struct fs_file *_file) @@ -369,10 +403,10 @@ static int fs_randomfail_write_stream_finish(struct fs_file *_file, bool success _file->output = NULL; else o_stream_unref(&_file->output); - } - if (!success || fs_random_fail(_file->fs, FS_OP_WRITE)) { - fs_write_stream_abort(file->super, &file->super_output); - return -1; + if (!success || fs_random_fail(_file->fs, 1, FS_OP_WRITE)) { + fs_write_stream_abort(file->super, &file->super_output); + return -1; + } } return fs_write_stream_finish(file->super, &file->super_output); } @@ -382,7 +416,7 @@ fs_randomfail_lock(struct fs_file *_file, unsigned int secs, struct fs_lock **lo { struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file; - if (fs_random_fail(_file->fs, FS_OP_LOCK)) + if (fs_random_fail(_file->fs, 1, FS_OP_LOCK)) return -1; return fs_lock(file->super, secs, lock_r); } @@ -395,52 +429,62 @@ static void fs_randomfail_unlock(struct fs_lock *_lock ATTR_UNUSED) static int fs_randomfail_exists(struct fs_file *_file) { struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file; + int ret; - if (fs_random_fail(_file->fs, FS_OP_EXISTS)) + if (fs_file_random_fail_begin(file, FS_OP_EXISTS)) return -1; - return fs_exists(file->super); + ret = fs_exists(file->super); + return fs_file_random_fail_end(file, ret, FS_OP_EXISTS); } static int fs_randomfail_stat(struct fs_file *_file, struct stat *st_r) { struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file; + int ret; - if (fs_random_fail(_file->fs, FS_OP_STAT)) + if (fs_file_random_fail_begin(file, FS_OP_STAT)) return -1; - return fs_stat(file->super, st_r); + ret = fs_stat(file->super, st_r); + return fs_file_random_fail_end(file, ret, FS_OP_STAT); } static int fs_randomfail_copy(struct fs_file *_src, struct fs_file *_dest) { struct randomfail_fs_file *src = (struct randomfail_fs_file *)_src; struct randomfail_fs_file *dest = (struct randomfail_fs_file *)_dest; + int ret; - if (fs_random_fail(_dest->fs, FS_OP_COPY)) + if (fs_file_random_fail_begin(dest, FS_OP_COPY)) return -1; if (_src != NULL) - return fs_copy(src->super, dest->super); + ret = fs_copy(src->super, dest->super); else - return fs_copy_finish_async(dest->super); + ret = fs_copy_finish_async(dest->super); + return fs_file_random_fail_end(dest, ret, FS_OP_COPY); } static int fs_randomfail_rename(struct fs_file *_src, struct fs_file *_dest) { struct randomfail_fs_file *src = (struct randomfail_fs_file *)_src; struct randomfail_fs_file *dest = (struct randomfail_fs_file *)_dest; + int ret; - if (fs_random_fail(_dest->fs, FS_OP_RENAME)) + if (fs_file_random_fail_begin(dest, FS_OP_RENAME)) return -1; - return fs_rename(src->super, dest->super); + ret = fs_rename(src->super, dest->super); + return fs_file_random_fail_end(dest, ret, FS_OP_RENAME); } static int fs_randomfail_delete(struct fs_file *_file) { struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file; + int ret; - if (fs_random_fail(_file->fs, FS_OP_DELETE)) + if (fs_file_random_fail_begin(file, FS_OP_DELETE)) return -1; - return fs_delete(file->super); + ret = fs_delete(file->super); + return fs_file_random_fail_end(file, ret, FS_OP_DELETE); } static struct fs_iter * @@ -523,6 +567,7 @@ const struct fs fs_class_randomfail = { fs_randomfail_delete, fs_randomfail_iter_init, fs_randomfail_iter_next, - fs_randomfail_iter_deinit + fs_randomfail_iter_deinit, + NULL } }; diff --git a/src/lib-fs/fs-sis-queue.c b/src/lib-fs/fs-sis-queue.c index 8259f0f0400..eaefb50e0ba 100644 --- a/src/lib-fs/fs-sis-queue.c +++ b/src/lib-fs/fs-sis-queue.c @@ -10,7 +10,6 @@ struct sis_queue_fs { struct fs fs; - struct fs *super; char *queue_dir; }; @@ -22,7 +21,7 @@ struct sis_queue_fs_file { static void fs_sis_queue_copy_error(struct sis_queue_fs *fs) { - fs_set_error(&fs->fs, "%s", fs_last_error(fs->super)); + fs_set_error(&fs->fs, "%s", fs_last_error(fs->fs.parent)); } static void fs_sis_queue_file_copy_error(struct sis_queue_fs_file *file) @@ -64,8 +63,8 @@ fs_sis_queue_init(struct fs *_fs, const char *args, parent_args = ""; else parent_name = t_strdup_until(parent_name, parent_args++); - if (fs_init(parent_name, parent_args, set, &fs->super, &error) < 0) { - fs_set_error(_fs, "%s: %s", parent_name, error); + if (fs_init(parent_name, parent_args, set, &_fs->parent, &error) < 0) { + fs_set_error(_fs, "%s", error); return -1; } return 0; @@ -75,17 +74,15 @@ static void fs_sis_queue_deinit(struct fs *_fs) { struct sis_queue_fs *fs = (struct sis_queue_fs *)_fs; - if (fs->super != NULL) - fs_deinit(&fs->super); + if (_fs->parent != NULL) + fs_deinit(&_fs->parent); i_free(fs->queue_dir); i_free(fs); } static enum fs_properties fs_sis_queue_get_properties(struct fs *_fs) { - struct sis_queue_fs *fs = (struct sis_queue_fs *)_fs; - - return fs_get_properties(fs->super); + return fs_get_properties(_fs->parent); } static struct fs_file * @@ -103,7 +100,7 @@ fs_sis_queue_file_init(struct fs *_fs, const char *path, if (mode == FS_OPEN_MODE_APPEND) fs_set_error(_fs, "APPEND mode not supported"); else - file->super = fs_file_init(fs->super, path, mode | flags); + file->super = fs_file_init(_fs->parent, path, mode | flags); return &file->file; } @@ -144,9 +141,7 @@ fs_sis_queue_set_async_callback(struct fs_file *_file, static int fs_sis_queue_wait_async(struct fs *_fs) { - struct sis_queue_fs *fs = (struct sis_queue_fs *)_fs; - - return fs_wait_async(fs->super); + return fs_wait_async(_fs->parent); } static void @@ -206,9 +201,9 @@ static void fs_sis_queue_add(struct sis_queue_fs_file *file) fname = path; queue_path = t_strdup_printf("%s/%s", fs->queue_dir, fname); - queue_file = fs_file_init(fs->super, queue_path, FS_OPEN_MODE_CREATE); + queue_file = fs_file_init(fs->fs.parent, queue_path, FS_OPEN_MODE_CREATE); if (fs_write(queue_file, "", 0) < 0 && errno != EEXIST) - i_error("fs-sis-queue: %s", fs_last_error(fs->super)); + i_error("fs-sis-queue: %s", fs_last_error(fs->fs.parent)); fs_file_deinit(&queue_file); } @@ -348,9 +343,7 @@ static struct fs_iter * fs_sis_queue_iter_init(struct fs *_fs, const char *path, enum fs_iter_flags flags) { - struct sis_queue_fs *fs = (struct sis_queue_fs *)_fs; - - return fs_iter_init(fs->super, path, flags); + return fs_iter_init(_fs->parent, path, flags); } const struct fs fs_class_sis_queue = { @@ -383,6 +376,7 @@ const struct fs fs_class_sis_queue = { fs_sis_queue_delete, fs_sis_queue_iter_init, NULL, + NULL, NULL } }; diff --git a/src/lib-fs/fs-sis.c b/src/lib-fs/fs-sis.c index bee09e7a96a..98ccf96b16b 100644 --- a/src/lib-fs/fs-sis.c +++ b/src/lib-fs/fs-sis.c @@ -12,7 +12,6 @@ struct sis_fs { struct fs fs; - struct fs *super; }; struct sis_fs_file { @@ -31,7 +30,7 @@ struct sis_fs_file { static void fs_sis_copy_error(struct sis_fs *fs) { - fs_set_error(&fs->fs, "%s", fs_last_error(fs->super)); + fs_set_error(&fs->fs, "%s", fs_last_error(fs->fs.parent)); } static void fs_sis_file_copy_error(struct sis_fs_file *file) @@ -53,7 +52,6 @@ static struct fs *fs_sis_alloc(void) static int fs_sis_init(struct fs *_fs, const char *args, const struct fs_settings *set) { - struct sis_fs *fs = (struct sis_fs *)_fs; enum fs_properties props; const char *parent_name, *parent_args, *error; @@ -70,11 +68,11 @@ fs_sis_init(struct fs *_fs, const char *args, const struct fs_settings *set) parent_name = t_strdup_until(args, parent_args); parent_args++; } - if (fs_init(parent_name, parent_args, set, &fs->super, &error) < 0) { - fs_set_error(_fs, "%s: %s", parent_name, error); + if (fs_init(parent_name, parent_args, set, &_fs->parent, &error) < 0) { + fs_set_error(_fs, "%s", error); return -1; } - props = fs_get_properties(fs->super); + props = fs_get_properties(_fs->parent); if ((props & FS_SIS_REQUIRED_PROPS) != FS_SIS_REQUIRED_PROPS) { fs_set_error(_fs, "%s backend can't be used with SIS", parent_name); @@ -87,16 +85,14 @@ static void fs_sis_deinit(struct fs *_fs) { struct sis_fs *fs = (struct sis_fs *)_fs; - if (fs->super != NULL) - fs_deinit(&fs->super); + if (_fs->parent != NULL) + fs_deinit(&_fs->parent); i_free(fs); } static enum fs_properties fs_sis_get_properties(struct fs *_fs) { - struct sis_fs *fs = (struct sis_fs *)_fs; - - return fs_get_properties(fs->super); + return fs_get_properties(_fs->parent); } static struct fs_file * @@ -124,7 +120,7 @@ fs_sis_file_init(struct fs *_fs, const char *path, /* if hashes/ already exists, open it */ file->hash_path = i_strdup_printf("%s/"HASH_DIR_NAME"/%s", dir, hash); - file->hash_file = fs_file_init(fs->super, file->hash_path, + file->hash_file = fs_file_init(_fs->parent, file->hash_path, FS_OPEN_MODE_READONLY); file->hash_input = fs_read_stream(file->hash_file, IO_BLOCK_SIZE); @@ -137,7 +133,7 @@ fs_sis_file_init(struct fs *_fs, const char *path, i_stream_destroy(&file->hash_input); } - file->super = fs_file_init(fs->super, path, mode | flags); + file->super = fs_file_init(_fs->parent, path, mode | flags); return &file->file; } @@ -181,9 +177,7 @@ fs_sis_set_async_callback(struct fs_file *_file, static int fs_sis_wait_async(struct fs *_fs) { - struct sis_fs *fs = (struct sis_fs *)_fs; - - return fs_wait_async(fs->super); + return fs_wait_async(_fs->parent); } static void @@ -486,9 +480,7 @@ static int fs_sis_delete(struct fs_file *_file) static struct fs_iter * fs_sis_iter_init(struct fs *_fs, const char *path, enum fs_iter_flags flags) { - struct sis_fs *fs = (struct sis_fs *)_fs; - - return fs_iter_init(fs->super, path, flags); + return fs_iter_init(_fs->parent, path, flags); } const struct fs fs_class_sis = { @@ -521,6 +513,7 @@ const struct fs fs_class_sis = { fs_sis_delete, fs_sis_iter_init, NULL, + NULL, NULL } }; diff --git a/src/lib-fs/fs-test-async.c b/src/lib-fs/fs-test-async.c new file mode 100644 index 00000000000..2a73a74c768 --- /dev/null +++ b/src/lib-fs/fs-test-async.c @@ -0,0 +1,97 @@ +/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "ostream.h" +#include "fs-test.h" +#include "test-common.h" + +static void test_fs_async_write(const char *test_name, struct fs *fs) +{ + struct fs_file *file; + struct test_fs_file *test_file; + struct ostream *output; + unsigned int i; + + test_begin(t_strdup_printf("%s: async write", test_name)); + for (i = 0; i < 3; i++) { + file = fs_file_init(fs, "foo", FS_OPEN_MODE_REPLACE | + FS_OPEN_FLAG_ASYNC); + output = fs_write_stream(file); + + o_stream_nsend_str(output, "12345"); + if (i < 2) { + test_assert(fs_write_stream_finish(file, &output) == 0); + test_assert(output == NULL); + test_assert(fs_write_stream_finish_async(file) == 0); + } + + test_file = test_fs_file_get(fs, "foo"); + test_file->wait_async = FALSE; + + switch (i) { + case 0: + test_assert(fs_write_stream_finish_async(file) > 0); + test_assert(test_file->contents->used > 0); + break; + case 1: + test_file->io_failure = TRUE; + test_assert(fs_write_stream_finish_async(file) < 0); + test_assert(test_file->contents->used == 0); + break; + case 2: + fs_write_stream_abort(file, &output); + test_assert(test_file->contents->used == 0); + break; + } + fs_file_deinit(&file); + } + test_end(); +} + +static void test_fs_async_copy(const char *test_name, struct fs *fs) +{ + struct fs_file *src, *dest; + struct test_fs_file *test_file; + + test_begin(t_strdup_printf("%s: async copy", test_name)); + + src = fs_file_init(fs, "foo", FS_OPEN_MODE_REPLACE); + test_assert(fs_write(src, "source", 6) == 0); + + dest = fs_file_init(fs, "bar", FS_OPEN_MODE_REPLACE | + FS_OPEN_FLAG_ASYNC); + + test_assert(fs_copy(src, dest) == -1 && errno == EAGAIN); + + test_file = test_fs_file_get(fs, "bar"); + test_file->wait_async = FALSE; + + test_assert(fs_copy_finish_async(dest) == 0); + test_assert(test_file->contents->used > 0); + fs_file_deinit(&dest); + + fs_file_deinit(&src); + test_end(); +} + +void test_fs_async(const char *test_name, enum fs_properties properties, + const char *driver, const char *args) +{ + struct fs_settings fs_set; + struct fs *fs; + struct test_fs *test_fs; + const char *error; + + memset(&fs_set, 0, sizeof(fs_set)); + if (fs_init(driver, args, &fs_set, &fs, &error) < 0) + i_fatal("fs_init() failed: %s", error); + + test_fs = test_fs_get(fs); + test_fs->properties = properties; + + test_fs_async_write(test_name, fs); + test_fs_async_copy(test_name, fs); + + fs_deinit(&fs); +} diff --git a/src/lib-fs/fs-test.c b/src/lib-fs/fs-test.c new file mode 100644 index 00000000000..b3c53a65f95 --- /dev/null +++ b/src/lib-fs/fs-test.c @@ -0,0 +1,423 @@ +/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "ostream.h" +#include "test-common.h" +#include "fs-test.h" + +static struct fs *fs_test_alloc(void) +{ + struct test_fs *fs; + + fs = i_new(struct test_fs, 1); + fs->fs = fs_class_test; + i_array_init(&fs->iter_files, 32); + return &fs->fs; +} + +static int +fs_test_init(struct fs *_fs ATTR_UNUSED, const char *args ATTR_UNUSED, + const struct fs_settings *set ATTR_UNUSED) +{ + return 0; +} + +static void fs_test_deinit(struct fs *_fs) +{ + struct test_fs *fs = (struct test_fs *)_fs; + + array_free(&fs->iter_files); + i_free(fs); +} + +static enum fs_properties fs_test_get_properties(struct fs *_fs) +{ + struct test_fs *fs = (struct test_fs *)_fs; + + return fs->properties; +} + +static struct fs_file * +fs_test_file_init(struct fs *_fs, const char *path, + enum fs_open_mode mode, enum fs_open_flags flags) +{ + struct test_fs_file *file; + + file = i_new(struct test_fs_file, 1); + file->file.fs = _fs; + file->file.path = i_strdup(path); + file->file.flags = flags; + file->mode = mode; + file->contents = buffer_create_dynamic(default_pool, 1024); + file->exists = TRUE; + file->seekable = TRUE; + file->wait_async = (flags & FS_OPEN_FLAG_ASYNC) != 0; + return &file->file; +} + +static void fs_test_file_deinit(struct fs_file *_file) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + buffer_free(&file->contents); + i_free(file->file.path); + i_free(file); +} + +static void fs_test_file_close(struct fs_file *_file) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + file->closed = TRUE; +} + +static const char *fs_test_file_get_path(struct fs_file *_file) +{ + return _file->path; +} + +static void +fs_test_set_async_callback(struct fs_file *_file, + fs_file_async_callback_t *callback, + void *context) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + file->async_callback = callback; + file->async_context = context; +} + +static int fs_test_wait_async(struct fs *_fs ATTR_UNUSED) +{ + return 0; +} + +static void +fs_test_set_metadata(struct fs_file *_file, const char *key, + const char *value) +{ + fs_default_set_metadata(_file, key, value); +} + +static int +fs_test_get_metadata(struct fs_file *_file, + const ARRAY_TYPE(fs_metadata) **metadata_r) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + if (file->wait_async) { + fs_set_error_async(_file->fs); + return -1; + } + if (file->io_failure) { + errno = EIO; + return -1; + } + fs_metadata_init(_file); + *metadata_r = &_file->metadata; + return 0; +} + +static bool fs_test_prefetch(struct fs_file *_file ATTR_UNUSED, + uoff_t length ATTR_UNUSED) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + file->prefetched = TRUE; + return TRUE; +} + +static void fs_test_stream_destroyed(struct test_fs_file *file) +{ + i_assert(file->input != NULL); + file->input = NULL; +} + +static struct istream * +fs_test_read_stream(struct fs_file *_file, size_t max_buffer_size ATTR_UNUSED) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + struct istream *input; + + i_assert(file->input == NULL); + + if (!file->exists) + return i_stream_create_error(ENOENT); + if (file->io_failure) + return i_stream_create_error(EIO); + input = test_istream_create_data(file->contents->data, + file->contents->used); + i_stream_add_destroy_callback(input, fs_test_stream_destroyed, file); + if (!file->seekable) + input->seekable = FALSE; + file->input = input; + return input; +} + +static void fs_test_write_stream(struct fs_file *_file) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + i_assert(_file->output == NULL); + + buffer_set_used_size(file->contents, 0); + _file->output = o_stream_create_buffer(file->contents); +} + +static int fs_test_write_stream_finish(struct fs_file *_file, bool success) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + if (_file->output != NULL) + o_stream_destroy(&_file->output); + if (file->wait_async) { + fs_set_error_async(_file->fs); + return 0; + } + if (file->io_failure) + success = FALSE; + if (!success) + buffer_set_used_size(file->contents, 0); + return success ? 1 : -1; +} + +static int +fs_test_lock(struct fs_file *_file, unsigned int secs ATTR_UNUSED, + struct fs_lock **lock_r) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + if (file->locked) + return 0; + file->locked = TRUE; + *lock_r = i_new(struct fs_lock, 1); + (*lock_r)->file = _file; + return 1; +} + +static void fs_test_unlock(struct fs_lock *lock) +{ + struct test_fs_file *file = (struct test_fs_file *)lock->file; + + file->locked = FALSE; + i_free(lock); +} + +static int fs_test_exists(struct fs_file *_file) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + if (file->wait_async) { + fs_set_error_async(_file->fs); + return -1; + } + if (file->io_failure) { + errno = EIO; + return -1; + } + return file->exists ? 1 : 0; +} + +static int fs_test_stat(struct fs_file *_file, struct stat *st_r) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + if (file->wait_async) { + fs_set_error_async(_file->fs); + return -1; + } + if (file->io_failure) { + errno = EIO; + return -1; + } + if (!file->exists) { + errno = ENOENT; + return -1; + } + memset(st_r, 0, sizeof(*st_r)); + st_r->st_size = file->contents->used; + return 0; +} + +static int fs_test_copy(struct fs_file *_src, struct fs_file *_dest) +{ + struct test_fs_file *src; + struct test_fs_file *dest = (struct test_fs_file *)_dest; + + if (_src != NULL) + dest->copy_src = test_fs_file_get(_src->fs, fs_file_path(_src)); + src = dest->copy_src; + if (dest->wait_async) { + fs_set_error_async(_dest->fs); + return -1; + } + dest->copy_src = NULL; + + if (dest->io_failure) { + errno = EIO; + return -1; + } + if (!src->exists) { + errno = ENOENT; + return -1; + } + buffer_set_used_size(dest->contents, 0); + buffer_append_buf(dest->contents, src->contents, 0, (size_t)-1); + dest->exists = TRUE; + return 0; +} + +static int fs_test_rename(struct fs_file *_src, struct fs_file *_dest) +{ + struct test_fs_file *src = (struct test_fs_file *)_src; + struct test_fs_file *dest = (struct test_fs_file *)_dest; + + if (src->wait_async || dest->wait_async) { + fs_set_error_async(_dest->fs); + return -1; + } + + if (fs_test_copy(_src, _dest) < 0) + return -1; + src->exists = FALSE; + return 0; +} + +static int fs_test_delete(struct fs_file *_file) +{ + struct test_fs_file *file = (struct test_fs_file *)_file; + + if (file->wait_async) { + fs_set_error_async(_file->fs); + return -1; + } + + if (!file->exists) { + errno = ENOENT; + return -1; + } + return 0; +} + +static struct fs_iter * +fs_test_iter_init(struct fs *_fs, const char *path, + enum fs_iter_flags flags) +{ + struct test_fs *fs = (struct test_fs *)_fs; + struct test_fs_iter *iter; + + iter = i_new(struct test_fs_iter, 1); + iter->iter.fs = _fs; + iter->iter.flags = flags; + iter->prefix = i_strdup(path); + iter->prefix_len = strlen(iter->prefix); + iter->prev_dir = i_strdup(""); + array_sort(&fs->iter_files, i_strcmp_p); + return &iter->iter; +} + +static const char *fs_test_iter_next(struct fs_iter *_iter) +{ + struct test_fs_iter *iter = (struct test_fs_iter *)_iter; + struct test_fs *fs = (struct test_fs *)_iter->fs; + const char *const *files, *p; + unsigned int count, len, prev_dir_len = strlen(iter->prev_dir); + + files = array_get(&fs->iter_files, &count); + for (; iter->idx < count; iter->idx++) { + const char *fname = files[iter->idx]; + + if (strncmp(fname, iter->prefix, iter->prefix_len) != 0) + continue; + p = strrchr(fname, '/'); + if ((_iter->flags & FS_ITER_FLAG_DIRS) == 0) { + if (p == NULL) + return fname; + if (p[1] == '\0') + continue; /* dir/ */ + return p+1; + } + + if (p == NULL) + continue; + len = p - fname; + if (len == 0) + continue; + if (len == prev_dir_len && + strncmp(fname, iter->prev_dir, len) == 0) + continue; + i_free(iter->prev_dir); + iter->prev_dir = i_strndup(fname, len); + return iter->prev_dir; + } + return NULL; +} + +static int fs_test_iter_deinit(struct fs_iter *_iter) +{ + struct test_fs_iter *iter = (struct test_fs_iter *)_iter; + int ret = iter->failed ? -1 : 0; + + i_free(iter->prefix); + i_free(iter); + return ret; +} + +struct test_fs *test_fs_get(struct fs *fs) +{ + while (strcmp(fs->name, "test") != 0) { + i_assert(fs->parent != NULL); + fs = fs->parent; + } + return (struct test_fs *)fs; +} + +struct test_fs_file *test_fs_file_get(struct fs *fs, const char *path) +{ + struct fs_file *file; + + fs = &test_fs_get(fs)->fs; + + for (file = fs->files;; file = file->next) { + i_assert(file != NULL); + if (strcmp(fs_file_path(file), path) == 0) + break; + } + return (struct test_fs_file *)file; +} + +const struct fs fs_class_test = { + .name = "test", + .v = { + fs_test_alloc, + fs_test_init, + fs_test_deinit, + fs_test_get_properties, + fs_test_file_init, + fs_test_file_deinit, + fs_test_file_close, + fs_test_file_get_path, + fs_test_set_async_callback, + fs_test_wait_async, + fs_test_set_metadata, + fs_test_get_metadata, + fs_test_prefetch, + NULL, + fs_test_read_stream, + NULL, + fs_test_write_stream, + fs_test_write_stream_finish, + fs_test_lock, + fs_test_unlock, + fs_test_exists, + fs_test_stat, + fs_test_copy, + fs_test_rename, + fs_test_delete, + fs_test_iter_init, + fs_test_iter_next, + fs_test_iter_deinit, + NULL + } +}; diff --git a/src/lib-fs/fs-test.h b/src/lib-fs/fs-test.h new file mode 100644 index 00000000000..5a2ed1434a6 --- /dev/null +++ b/src/lib-fs/fs-test.h @@ -0,0 +1,45 @@ +#ifndef FS_TEST_H +#define FS_TEST_H + +#include "fs-api-private.h" + +struct test_fs { + struct fs fs; + enum fs_properties properties; + ARRAY_TYPE(const_string) iter_files; +}; + +struct test_fs_file { + struct fs_file file; + enum fs_open_mode mode; + + fs_file_async_callback_t *async_callback; + void *async_context; + + buffer_t *contents; + struct istream *input; + struct test_fs_file *copy_src; + + bool prefetched; + bool locked; + bool exists; + bool seekable; + bool closed; + bool io_failure; + bool wait_async; +}; + +struct test_fs_iter { + struct fs_iter iter; + char *prefix, *prev_dir; + unsigned int prefix_len, idx; + bool failed; +}; + +struct test_fs *test_fs_get(struct fs *fs); +struct test_fs_file *test_fs_file_get(struct fs *fs, const char *path); + +void test_fs_async(const char *test_name, enum fs_properties properties, + const char *driver, const char *args); + +#endif diff --git a/src/lib-fs/istream-fs-file.c b/src/lib-fs/istream-fs-file.c index 4dc5c2d153c..e7aecb3c13e 100644 --- a/src/lib-fs/istream-fs-file.c +++ b/src/lib-fs/istream-fs-file.c @@ -27,7 +27,7 @@ static ssize_t i_stream_fs_file_read(struct istream_private *stream) if (fstream->istream.parent == NULL) { input = fs_read_stream(fstream->file, - fstream->istream.max_buffer_size); + i_stream_get_max_buffer_size(&stream->istream)); i_stream_init_parent(stream, input); i_stream_unref(&input); } diff --git a/src/lib-fs/istream-metawrap.c b/src/lib-fs/istream-metawrap.c index c389432d7df..5f4528136e4 100644 --- a/src/lib-fs/istream-metawrap.c +++ b/src/lib-fs/istream-metawrap.c @@ -4,6 +4,8 @@ #include "istream-private.h" #include "istream-metawrap.h" +#define METAWRAP_MAX_METADATA_LINE_LEN 8192 + struct metawrap_istream { struct istream_private istream; metawrap_callback_t *callback; @@ -25,7 +27,8 @@ static int metadata_header_read(struct metawrap_istream *mstream) p = strchr(line, ':'); if (p == NULL) { io_stream_set_error(&mstream->istream.iostream, - "Metadata header line is missing ':'"); + "Metadata header line is missing ':' at offset %"PRIuUOFF_T, + mstream->istream.istream.v_offset); mstream->istream.istream.stream_errno = EINVAL; return -1; } @@ -33,8 +36,16 @@ static int metadata_header_read(struct metawrap_istream *mstream) mstream->callback(line, p, mstream->context); } if (mstream->istream.parent->eof) { - mstream->istream.istream.stream_errno = - mstream->istream.parent->stream_errno; + if (mstream->istream.parent->stream_errno != 0) { + mstream->istream.istream.stream_errno = + mstream->istream.parent->stream_errno; + } else { + io_stream_set_error(&mstream->istream.iostream, + "Metadata header is missing ending line at offset %"PRIuUOFF_T, + mstream->istream.istream.v_offset); + mstream->istream.istream.stream_errno = EINVAL; + return -1; + } mstream->istream.istream.eof = TRUE; return -1; } @@ -51,7 +62,12 @@ static ssize_t i_stream_metawrap_read(struct istream_private *stream) stream->istream.v_offset); if (mstream->in_metadata) { + size_t prev_max_size = i_stream_get_max_buffer_size(stream->parent); + + i_stream_set_max_buffer_size(stream->parent, METAWRAP_MAX_METADATA_LINE_LEN); ret = metadata_header_read(mstream); + i_stream_set_max_buffer_size(stream->parent, prev_max_size); + i_assert(stream->istream.v_offset == 0); mstream->start_offset = stream->parent->v_offset; if (ret <= 0) @@ -125,7 +141,9 @@ i_stream_create_metawrap(struct istream *input, mstream->istream.seek = i_stream_metawrap_seek; mstream->istream.stat = input->seekable ? i_stream_metawrap_stat : NULL; - mstream->istream.istream.readable_fd = input->readable_fd; + /* we can't set abs_start_offset early enough so that it would get + passed to our child istreams. */ + mstream->istream.istream.readable_fd = FALSE; mstream->istream.istream.blocking = input->blocking; mstream->istream.istream.seekable = input->seekable; mstream->in_metadata = TRUE; diff --git a/src/lib-fs/test-fs-metawrap.c b/src/lib-fs/test-fs-metawrap.c new file mode 100644 index 00000000000..6b1f52db7a7 --- /dev/null +++ b/src/lib-fs/test-fs-metawrap.c @@ -0,0 +1,64 @@ +/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "istream.h" +#include "fs-test.h" +#include "test-common.h" + +static void test_fs_metawrap_stat(void) +{ + struct fs_settings fs_set; + struct fs *fs; + struct fs_file *file; + struct test_fs_file *test_file; + struct istream *input; + struct stat st; + const char *error; + unsigned int i; + + test_begin("fs metawrap stat"); + + memset(&fs_set, 0, sizeof(fs_set)); + if (fs_init("metawrap", "test", &fs_set, &fs, &error) < 0) + i_fatal("fs_init() failed: %s", error); + + for (i = 0; i < 2; i++) { + file = fs_file_init(fs, "foo", FS_OPEN_MODE_READONLY); + + test_file = test_fs_file_get(fs, "foo"); + str_append(test_file->contents, "key:value\n\n12345678901234567890"); + + if (i == 0) { + input = fs_read_stream(file, 2); + test_istream_set_max_buffer_size(test_file->input, 2); + } else { + input = NULL; + } + + test_assert_idx(fs_stat(file, &st) == 0 && st.st_size == 20, i); + + if (input != NULL) + i_stream_unref(&input); + fs_file_deinit(&file); + } + fs_deinit(&fs); + test_end(); +} + +static void test_fs_metawrap_async(void) +{ + test_fs_async("metawrap", FS_PROPERTY_METADATA, "metawrap", "test"); + test_fs_async("metawrap passthrough", 0, "metawrap", "test"); + test_fs_async("double-metawrap", FS_PROPERTY_METADATA, "metawrap", "metawrap:test"); +} + +int main(void) +{ + static void (*test_functions[])(void) = { + test_fs_metawrap_stat, + test_fs_metawrap_async, + NULL + }; + return test_run(test_functions); +} diff --git a/src/lib-fts/Makefile.am b/src/lib-fts/Makefile.am index fa04b2bad05..1219ddf82a9 100644 --- a/src/lib-fts/Makefile.am +++ b/src/lib-fts/Makefile.am @@ -12,15 +12,23 @@ AM_CPPFLAGS = \ $(LIBICU_CFLAGS) \ -DUDHRDIR=\""$(top_srcdir)/src/lib-fts"\" \ -DDATADIR=\"$(pkgdatadir)\" \ - -DTEST_STOPWORDS_DIR=\""$(top_srcdir)/src/lib-fts"\" + -DTEST_STOPWORDS_DIR=\""$(top_srcdir)/src/lib-fts/stopwords"\" stopwordsdir = $(datadir)/${PACKAGE_TARNAME}/stopwords dist_stopwords_DATA = \ - stopwords_en.txt \ - stopwords_fi.txt \ - stopwords_fr.txt \ - stopwords_no.txt \ - stopwords_sv.txt + stopwords/stopwords_da.txt \ + stopwords/stopwords_de.txt \ + stopwords/stopwords_en.txt \ + stopwords/stopwords_es.txt \ + stopwords/stopwords_fi.txt \ + stopwords/stopwords_fr.txt \ + stopwords/stopwords_it.txt \ + stopwords/stopwords_nl.txt \ + stopwords/stopwords_no.txt \ + stopwords/stopwords_pt.txt \ + stopwords/stopwords_ro.txt \ + stopwords/stopwords_ru.txt \ + stopwords/stopwords_sv.txt BUILT_SOURCES = word-boundary-data.c word-break-data.c @@ -30,7 +38,8 @@ EXTRA_DIST = \ word-properties.pl \ WordBreakProperty.txt \ word-boundary-data.c \ - word-break-data.c + word-break-data.c \ + stopwords/stopwords_malformed.txt WordBreakProperty.txt: test -f WordBreakProperty.txt || wget http://www.unicode.org/Public/UNIDATA/auxiliary/WordBreakProperty.txt diff --git a/src/lib-fts/fts-filter-normalizer-icu.c b/src/lib-fts/fts-filter-normalizer-icu.c index 4a9cf9fbc86..58b2a9b475f 100644 --- a/src/lib-fts/fts-filter-normalizer-icu.c +++ b/src/lib-fts/fts-filter-normalizer-icu.c @@ -4,6 +4,7 @@ #include "buffer.h" #include "str.h" #include "unichar.h" /* unicode replacement char */ +#include "fts-tokenizer-common.h" #include "fts-filter-private.h" #include "fts-language.h" @@ -18,6 +19,7 @@ struct fts_filter_normalizer_icu { UTransliterator *transliterator; buffer_t *utf16_token, *trans_token; string_t *utf8_token; + unsigned int maxlen; }; static void fts_filter_normalizer_icu_destroy(struct fts_filter *filter) @@ -38,7 +40,7 @@ fts_filter_normalizer_icu_create(const struct fts_language *lang ATTR_UNUSED, { struct fts_filter_normalizer_icu *np; pool_t pp; - unsigned int i; + unsigned int i, max_length = 250; const char *id = "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC; [\\x20] Remove"; for (i = 0; settings[i] != NULL; i += 2) { @@ -46,6 +48,12 @@ fts_filter_normalizer_icu_create(const struct fts_language *lang ATTR_UNUSED, if (strcmp(key, "id") == 0) { id = value; + } else if (strcmp(key, "maxlen") == 0) { + if (str_to_uint(value, &max_length) < 0 || + max_length == 0) { + *error_r = t_strdup_printf("Invalid icu maxlen setting: %s", value); + return -1; + } } else { *error_r = t_strdup_printf("Unknown setting: %s", key); return -1; @@ -61,6 +69,7 @@ fts_filter_normalizer_icu_create(const struct fts_language *lang ATTR_UNUSED, np->utf16_token = buffer_create_dynamic(pp, 128); np->trans_token = buffer_create_dynamic(pp, 128); np->utf8_token = buffer_create_dynamic(pp, 128); + np->maxlen = max_length; *filter_r = &np->filter; return 0; } @@ -92,6 +101,11 @@ fts_filter_normalizer_icu_filter(struct fts_filter *filter, const char **token, fts_icu_utf16_to_utf8(np->utf8_token, np->trans_token->data, np->trans_token->used / sizeof(UChar)); + if (str_len(np->utf8_token) > np->maxlen) { + size_t len = np->maxlen; + fts_tokenizer_delete_trailing_partial_char(np->utf8_token->data, &len); + str_truncate(np->utf8_token, len); + } *token = str_c(np->utf8_token); return 1; } diff --git a/src/lib-fts/fts-filter-stopwords.c b/src/lib-fts/fts-filter-stopwords.c index 2d6df6964f9..8b7a1067a55 100644 --- a/src/lib-fts/fts-filter-stopwords.c +++ b/src/lib-fts/fts-filter-stopwords.c @@ -11,8 +11,8 @@ #define STOPWORDS_FILE_FORMAT "%s/stopwords_%s.txt" -#define STOPWORDS_COMMENT_CHAR1 '|' -#define STOPWORDS_COMMENT_CHAR2 '#' +#define STOPWORDS_CUTCHARS "|#\t " +#define STOPWORDS_DISALLOWED_CHARS "/\\<>.,\":()\t\n\r" struct fts_filter_stopwords { struct fts_filter filter; @@ -26,29 +26,33 @@ static int fts_filter_stopwords_read_list(struct fts_filter_stopwords *filter, const char **error_r) { struct istream *input; - const char *line, **words, *path; + const char *line, *word, *path; int ret = 0; + size_t len; path = t_strdup_printf(STOPWORDS_FILE_FORMAT, filter->stopwords_dir, filter->lang->name); input = i_stream_create_file(path, IO_BLOCK_SIZE); - while ((line = i_stream_read_next_line(input)) != NULL) T_BEGIN { - line = t_strcut(line, STOPWORDS_COMMENT_CHAR1); - line = t_strcut(line, STOPWORDS_COMMENT_CHAR2); - - words = t_strsplit_spaces(line, " \t"); - for (; *words != NULL; words++) { - const char *word = p_strdup(filter->pool, *words); - hash_table_insert(filter->stopwords, word, word); - } - } T_END; + while ((line = i_stream_read_next_line(input)) != NULL) { + len = strcspn(line, STOPWORDS_CUTCHARS); + if (len == 0) + continue; + if (strcspn(line, STOPWORDS_DISALLOWED_CHARS) < len) + continue; + word = p_strndup(filter->pool, line, len); + hash_table_insert(filter->stopwords, word, word); + } if (input->stream_errno != 0) { *error_r = t_strdup_printf("Failed to read stopword list %s: %s", path, i_stream_get_error(input)); ret = -1; } + + if (ret == 0 && hash_table_count(filter->stopwords) == 0) + i_warning("Stopwords list \"%s\" seems empty. Is the file correctly formatted?", path); + i_stream_destroy(&input); return ret; } diff --git a/src/lib-fts/fts-filter.h b/src/lib-fts/fts-filter.h index e37bdff6656..0c89f2566e6 100644 --- a/src/lib-fts/fts-filter.h +++ b/src/lib-fts/fts-filter.h @@ -24,8 +24,10 @@ extern const struct fts_filter *fts_filter_stemmer_snowball; Settings: "id", description of the normalizing/translitterating rules to use. See http://userguide.icu-project.org/transforms/general#TOC-Transliterator-Identifiers - for syntax. Defaults to "Any-Lower; NFKD; [: Nonspacing Mark :] - Remove; NFC" + for syntax. Defaults to "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC" + + "maxlen", maximum length of tokens that ICU normalizer will output. + Defaults to 250. */ extern const struct fts_filter *fts_filter_normalizer_icu; diff --git a/src/lib-fts/fts-tokenizer-address.c b/src/lib-fts/fts-tokenizer-address.c index 96f10eccddc..2b9380c499a 100644 --- a/src/lib-fts/fts-tokenizer-address.c +++ b/src/lib-fts/fts-tokenizer-address.c @@ -79,6 +79,9 @@ static void fts_tokenizer_address_current_token(struct email_address_fts_tokenizer *tok, const char **token_r) { + const unsigned char *data = tok->last_word->data; + size_t len = tok->last_word->used; + tok->tokenizer.skip_parents = TRUE; tok->state = EMAIL_ADDRESS_PARSER_STATE_NONE; if (str_len(tok->last_word) > tok->max_length) { @@ -86,15 +89,15 @@ fts_tokenizer_address_current_token(struct email_address_fts_tokenizer *tok, /* As future proofing, delete partial utf8. IS_DTEXT() does not actually allow utf8 addresses yet though. */ - const unsigned char *data = tok->last_word->data; - size_t len = tok->last_word->used; + len = tok->last_word->used; fts_tokenizer_delete_trailing_partial_char(data, &len); i_assert(len <= tok->max_length); - *token_r = len == 0 ? "" : - t_strndup(tok->last_word->data, len); - } else { - *token_r = t_strdup(str_c(tok->last_word)); } + + if (len > 0) + fts_tokenizer_delete_trailing_invalid_char(data, &len); + *token_r = len == 0 ? "" : + t_strndup(data, len); } static bool @@ -186,10 +189,10 @@ fts_tokenizer_email_address_parse_domain(struct email_address_fts_tokenizer *tok { size_t pos = 0; - while (pos < size && (IS_DTEXT(data[pos]) || data[pos] == '.')) + while (pos < size && (IS_DTEXT(data[pos]) || data[pos] == '.' || data[pos] == '-')) pos++; /* A complete domain name */ - if ((pos > 1 && pos < size) || /* non-atext after atext in this data*/ + if ((pos > 0 && pos < size) || /* non-atext after atext in this data*/ (pos < size && !domain_is_empty(tok))) { /* non-atext after previous atext */ str_append_n(tok->last_word, data, pos); *skip_r = pos; diff --git a/src/lib-fts/fts-tokenizer-common.c b/src/lib-fts/fts-tokenizer-common.c index f71113d036d..92feb71c28a 100644 --- a/src/lib-fts/fts-tokenizer-common.c +++ b/src/lib-fts/fts-tokenizer-common.c @@ -20,3 +20,14 @@ fts_tokenizer_delete_trailing_partial_char(const unsigned char *data, *len = pos; } } +void fts_tokenizer_delete_trailing_invalid_char(const unsigned char *data, + size_t *len) +{ + size_t pos = *len; + + /* the token may contain '.' in the end - remove all of them. */ + while (pos > 0 && + (data[pos-1] == '.' || data[pos-1] == '-')) + pos--; + *len = pos; +} diff --git a/src/lib-fts/fts-tokenizer-common.h b/src/lib-fts/fts-tokenizer-common.h index fdd3b163137..b90e54353e9 100644 --- a/src/lib-fts/fts-tokenizer-common.h +++ b/src/lib-fts/fts-tokenizer-common.h @@ -3,4 +3,7 @@ void fts_tokenizer_delete_trailing_partial_char(const unsigned char *data, size_t *len); +void +fts_tokenizer_delete_trailing_invalid_char(const unsigned char *data, + size_t *len); #endif diff --git a/src/lib-fts/stopwords/stopwords_da.txt b/src/lib-fts/stopwords/stopwords_da.txt new file mode 100644 index 00000000000..c4cfd9cf04a --- /dev/null +++ b/src/lib-fts/stopwords/stopwords_da.txt @@ -0,0 +1,109 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/danish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + + | A Danish stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This is a ranked list (commonest to rarest) of stopwords derived from + | a large text sample. + + +og | and +i | in +jeg | I +det | that (dem. pronoun)/it (pers. pronoun) +at | that (in front of a sentence)/to (with infinitive) +en | a/an +den | it (pers. pronoun)/that (dem. pronoun) +til | to/at/for/until/against/by/of/into, more +er | present tense of "to be" +som | who, as +på | on/upon/in/on/at/to/after/of/with/for, on +de | they +med | with/by/in, along +han | he +af | of/by/from/off/for/in/with/on, off +for | at/for/to/from/by/of/ago, in front/before, because +ikke | not +der | who/which, there/those +var | past tense of "to be" +mig | me/myself +sig | oneself/himself/herself/itself/themselves +men | but +et | a/an/one, one (number), someone/somebody/one +har | present tense of "to have" +om | round/about/for/in/a, about/around/down, if +vi | we +min | my +havde | past tense of "to have" +ham | him +hun | she +nu | now +over | over/above/across/by/beyond/past/on/about, over/past +da | then, when/as/since +fra | from/off/since, off, since +du | you +ud | out +sin | his/her/its/one's +dem | them +os | us/ourselves +op | up +man | you/one +hans | his +hvor | where +eller | or +hvad | what +skal | must/shall etc. +selv | myself/youself/herself/ourselves etc., even +her | here +alle | all/everyone/everybody etc. +vil | will (verb) +blev | past tense of "to stay/to remain/to get/to become" +kunne | could +ind | in +når | when +være | present tense of "to be" +dog | however/yet/after all +noget | something +ville | would +jo | you know/you see (adv), yes +deres | their/theirs +efter | after/behind/according to/for/by/from, later/afterwards +ned | down +skulle | should +denne | this +end | than +dette | this +mit | my/mine +også | also +under | under/beneath/below/during, below/underneath +have | have +dig | you +anden | other +hende | her +mine | my +alt | everything +meget | much/very, plenty of +sit | his, her, its, one's +sine | his, her, its, one's +vor | our +mod | against +disse | these +hvis | if +din | your/yours +nogle | some +hos | by/at +blive | be/become +mange | many +ad | by/through +bliver | present tense of "to be/to become" +hendes | her/hers +været | be +thi | for (conj) +jer | you +sådan | such, like this/like that diff --git a/src/lib-fts/stopwords/stopwords_de.txt b/src/lib-fts/stopwords/stopwords_de.txt new file mode 100644 index 00000000000..786284d7a58 --- /dev/null +++ b/src/lib-fts/stopwords/stopwords_de.txt @@ -0,0 +1,293 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/german/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + + | A German stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | The number of forms in this list is reduced significantly by passing it + | through the German stemmer. + + +aber | but + +alle | all +allem +allen +aller +alles + +als | than, as +also | so +am | an + dem +an | at + +ander | other +andere +anderem +anderen +anderer +anderes +anderm +andern +anderr +anders + +auch | also +auf | on +aus | out of +bei | by +bin | am +bis | until +bist | art +da | there +damit | with it +dann | then + +der | the +den +des +dem +die +das + +daß | that + +derselbe | the same +derselben +denselben +desselben +demselben +dieselbe +dieselben +dasselbe + +dazu | to that + +dein | thy +deine +deinem +deinen +deiner +deines + +denn | because + +derer | of those +dessen | of him + +dich | thee +dir | to thee +du | thou + +dies | this +diese +diesem +diesen +dieser +dieses + + +doch | (several meanings) +dort | (over) there + + +durch | through + +ein | a +eine +einem +einen +einer +eines + +einig | some +einige +einigem +einigen +einiger +einiges + +einmal | once + +er | he +ihn | him +ihm | to him + +es | it +etwas | something + +euer | your +eure +eurem +euren +eurer +eures + +für | for +gegen | towards +gewesen | p.p. of sein +hab | have +habe | have +haben | have +hat | has +hatte | had +hatten | had +hier | here +hin | there +hinter | behind + +ich | I +mich | me +mir | to me + + +ihr | you, to her +ihre +ihrem +ihren +ihrer +ihres +euch | to you + +im | in + dem +in | in +indem | while +ins | in + das +ist | is + +jede | each, every +jedem +jeden +jeder +jedes + +jene | that +jenem +jenen +jener +jenes + +jetzt | now +kann | can + +kein | no +keine +keinem +keinen +keiner +keines + +können | can +könnte | could +machen | do +man | one + +manche | some, many a +manchem +manchen +mancher +manches + +mein | my +meine +meinem +meinen +meiner +meines + +mit | with +muss | must +musste | had to +nach | to(wards) +nicht | not +nichts | nothing +noch | still, yet +nun | now +nur | only +ob | whether +oder | or +ohne | without +sehr | very + +sein | his +seine +seinem +seinen +seiner +seines + +selbst | self +sich | herself + +sie | they, she +ihnen | to them + +sind | are +so | so + +solche | such +solchem +solchen +solcher +solches + +soll | shall +sollte | should +sondern | but +sonst | else +über | over +um | about, around +und | and + +uns | us +unse +unsem +unsen +unser +unses + +unter | under +viel | much +vom | von + dem +von | from +vor | before +während | while +war | was +waren | were +warst | wast +was | what +weg | away, off +weil | because +weiter | further + +welche | which +welchem +welchen +welcher +welches + +wenn | when +werde | will +werden | will +wie | how +wieder | again +will | want +wir | we +wird | will +wirst | willst +wo | where +wollen | want +wollte | wanted +würde | would +würden | would +zu | to +zum | zu + dem +zur | zu + der +zwar | indeed +zwischen | between + diff --git a/src/lib-fts/stopwords_en.txt b/src/lib-fts/stopwords/stopwords_en.txt similarity index 100% rename from src/lib-fts/stopwords_en.txt rename to src/lib-fts/stopwords/stopwords_en.txt diff --git a/src/lib-fts/stopwords/stopwords_es.txt b/src/lib-fts/stopwords/stopwords_es.txt new file mode 100644 index 00000000000..ed9fdca9c2f --- /dev/null +++ b/src/lib-fts/stopwords/stopwords_es.txt @@ -0,0 +1,355 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/spanish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + + | A Spanish stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + + | The following is a ranked list (commonest to rarest) of stopwords + | deriving from a large sample of text. + + | Extra words have been added at the end. + +de | from, of +la | the, her +que | who, that +el | the +en | in +y | and +a | to +los | the, them +del | de + el +se | himself, from him etc +las | the, them +por | for, by, etc +un | a +para | for +con | with +no | no +una | a +su | his, her +al | a + el + | es from SER +lo | him +como | how +más | more +pero | pero +sus | su plural +le | to him, her +ya | already +o | or + | fue from SER +este | this + | ha from HABER +sí | himself etc +porque | because +esta | this + | son from SER +entre | between + | está from ESTAR +cuando | when +muy | very +sin | without +sobre | on + | ser from SER + | tiene from TENER +también | also +me | me +hasta | until +hay | there is/are +donde | where + | han from HABER +quien | whom, that + | están from ESTAR + | estado from ESTAR +desde | from +todo | all +nos | us +durante | during + | estados from ESTAR +todos | all +uno | a +les | to them +ni | nor +contra | against +otros | other + | fueron from SER +ese | that +eso | that + | había from HABER +ante | before +ellos | they +e | and (variant of y) +esto | this +mí | me +antes | before +algunos | some +qué | what? +unos | a +yo | I +otro | other +otras | other +otra | other +él | he +tanto | so much, many +esa | that +estos | these +mucho | much, many +quienes | who +nada | nothing +muchos | many +cual | who + | sea from SER +poco | few +ella | she +estar | to be + | haber from HABER +estas | these + | estaba from ESTAR + | estamos from ESTAR +algunas | some +algo | something +nosotros | we + + | other forms + +mi | me +mis | mi plural +tú | thou +te | thee +ti | thee +tu | thy +tus | tu plural +ellas | they +nosotras | we +vosotros | you +vosotras | you +os | you +mío | mine +mía | +míos | +mías | +tuyo | thine +tuya | +tuyos | +tuyas | +suyo | his, hers, theirs +suya | +suyos | +suyas | +nuestro | ours +nuestra | +nuestros | +nuestras | +vuestro | yours +vuestra | +vuestros | +vuestras | +esos | those +esas | those + + | forms of estar, to be (not including the infinitive): +estoy +estás +está +estamos +estáis +están +esté +estés +estemos +estéis +estén +estaré +estarás +estará +estaremos +estaréis +estarán +estaría +estarías +estaríamos +estaríais +estarían +estaba +estabas +estábamos +estabais +estaban +estuve +estuviste +estuvo +estuvimos +estuvisteis +estuvieron +estuviera +estuvieras +estuviéramos +estuvierais +estuvieran +estuviese +estuvieses +estuviésemos +estuvieseis +estuviesen +estando +estado +estada +estados +estadas +estad + + | forms of haber, to have (not including the infinitive): +he +has +ha +hemos +habéis +han +haya +hayas +hayamos +hayáis +hayan +habré +habrás +habrá +habremos +habréis +habrán +habría +habrías +habríamos +habríais +habrían +había +habías +habíamos +habíais +habían +hube +hubiste +hubo +hubimos +hubisteis +hubieron +hubiera +hubieras +hubiéramos +hubierais +hubieran +hubiese +hubieses +hubiésemos +hubieseis +hubiesen +habiendo +habido +habida +habidos +habidas + + | forms of ser, to be (not including the infinitive): +soy +eres +es +somos +sois +son +sea +seas +seamos +seáis +sean +seré +serás +será +seremos +seréis +serán +sería +serías +seríamos +seríais +serían +era +eras +éramos +erais +eran +fui +fuiste +fue +fuimos +fuisteis +fueron +fuera +fueras +fuéramos +fuerais +fueran +fuese +fueses +fuésemos +fueseis +fuesen +siendo +sido + | sed also means 'thirst' + + | forms of tener, to have (not including the infinitive): +tengo +tienes +tiene +tenemos +tenéis +tienen +tenga +tengas +tengamos +tengáis +tengan +tendré +tendrás +tendrá +tendremos +tendréis +tendrán +tendría +tendrías +tendríamos +tendríais +tendrían +tenía +tenías +teníamos +teníais +tenían +tuve +tuviste +tuvo +tuvimos +tuvisteis +tuvieron +tuviera +tuvieras +tuviéramos +tuvierais +tuvieran +tuviese +tuvieses +tuviésemos +tuvieseis +tuviesen +teniendo +tenido +tenida +tenidos +tenidas +tened + diff --git a/src/lib-fts/stopwords/stopwords_fi.txt b/src/lib-fts/stopwords/stopwords_fi.txt new file mode 100644 index 00000000000..95c74f256b8 --- /dev/null +++ b/src/lib-fts/stopwords/stopwords_fi.txt @@ -0,0 +1,260 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/finnish/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + +| forms of BE + +olla +olen +olet +on +olemme +olette +ovat +ole | negative form + +oli +olisi +olisit +olisin +olisimme +olisitte +olisivat +olit +olin +olimme +olitte +olivat +ollut +olleet + +en | negation +et +ei +emme +ette +eivät + +|Words below in the following cases: +|Nom Gen Acc Part Iness Elat Illat Adess Ablat Allat Ess Trans +minä | I +minun | I +minut | I +minua | I +minussa | I +minusta | I +minuun | I +minulla | I +minulta | I +minulle | I +sinä | you +sinun | you +sinut | you +sinua | you +sinussa | you +sinusta | you +sinuun | you +sinulla | you +sinulta | you +sinulle | you +hän | he she +hänen | he she +hänet | he she +häntä | he she +hänessä | he she +hänestä | he she +häneen | he she +hänellä | he she +häneltä | he she +hänelle | he she +me | we +meidän | we +meidät | we +meitä | we +meissä | we +meistä | we +meihin | we +meillä | we +meiltä | we +meille | we +te | you +teidän | you +teidät | you +teitä | you +teissä | you +teistä | you +teihin | you +teillä | you +teiltä | you +teille | you +he | they +heidän | they +heidät | they +heitä | they +heissä | they +heistä | they +heihin | they +heillä | they +heiltä | they +heille | they +tämä | this +tämän | this +tätä | this +tässä | this +tästä | this +tähän | this +tallä | this +tältä | this +tälle | this +tänä | this +täksi | this +tuo | that +tuon | that +tuotä | that +tuossa | that +tuosta | that +tuohon | that +tuolla | that +tuolta | that +tuolle | that +tuona | that +tuoksi | that +se | it +sen | it +sitä | it +siinä | it +siitä | it +siihen | it +sillä | it +siltä | it +sille | it +sinä | it +siksi | it +nämä | these +näiden | these +näitä | these +näissä | these +näistä | these +näihin | these +näillä | these +näiltä | these +näille | these +näinä | these +näiksi | these +nuo | those +noiden | those +noita | those +noissa | those +noista | those +noihin | those +noilla | those +noilta | those +noille | those +noina | those +noiksi | those +ne | they +niiden | they +niitä | they +niissä | they +niistä | they +niihin | they +niillä | they +niiltä | they +niille | they +niinä | they +niiksi | they +kuka | who +kenen | who +kenet | who +ketä | who +kenessä | who +kenestä | who +keneen | who +kenellä | who +keneltä | who +kenelle | who +kenenä | who +keneksi | who +ketkä | who (pl) +keiden | who (pl) +ketkä | who (pl) +keitä | who (pl) +keissä | who (pl) +keistä | who (pl) +keihin | who (pl) +keillä | who (pl) +keiltä | who (pl) +keille | who (pl) +keinä | who (pl) +keiksi | who (pl) +mikä | which what +minkä | which what +minkä | which what +mitä | which what +missä | which what +mistä | which what +mihin | which what +millä | which what +miltä | which what +mille | which what +minä | which what +miksi | which what +mitkä | which what (pl) +joka | who which +jonka | who which +jota | who which +jossa | who which +josta | who which +johon | who which +jolla | who which +jolta | who which +jolle | who which +jona | who which +joksi | who which +jotka | who which (pl) +joiden | who which (pl) +joita | who which (pl) +joissa | who which (pl) +joista | who which (pl) +joihin | who which (pl) +joilla | who which (pl) +joilta | who which (pl) +joille | who which (pl) +joina | who which (pl) +joiksi | who which (pl) + +| conjunctions + +että | that +ja | and +jos | if +koska | because +kuin | than +mutta | but +niin | so +sekä | and +sillä | for +tai | or +vaan | but +vai | or +vaikka | although + + +| prepositions + +kanssa | with +mukaan | according to +noin | about +poikki | across +yli | over, across + +| other + +kun | when +niin | so +nyt | now +itse | self + diff --git a/src/lib-fts/stopwords_fr.txt b/src/lib-fts/stopwords/stopwords_fr.txt similarity index 100% rename from src/lib-fts/stopwords_fr.txt rename to src/lib-fts/stopwords/stopwords_fr.txt diff --git a/src/lib-fts/stopwords/stopwords_it.txt b/src/lib-fts/stopwords/stopwords_it.txt new file mode 100644 index 00000000000..a9e5f676344 --- /dev/null +++ b/src/lib-fts/stopwords/stopwords_it.txt @@ -0,0 +1,302 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/italian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + + | An Italian stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + +ad | a (to) before vowel +al | a + il +allo | a + lo +ai | a + i +agli | a + gli +all | a + l' +agl | a + gl' +alla | a + la +alle | a + le +con | with +col | con + il +coi | con + i (forms collo, cogli etc are now very rare) +da | from +dal | da + il +dallo | da + lo +dai | da + i +dagli | da + gli +dall | da + l' +dagl | da + gll' +dalla | da + la +dalle | da + le +di | of +del | di + il +dello | di + lo +dei | di + i +degli | di + gli +dell | di + l' +degl | di + gl' +della | di + la +delle | di + le +in | in +nel | in + el +nello | in + lo +nei | in + i +negli | in + gli +nell | in + l' +negl | in + gl' +nella | in + la +nelle | in + le +su | on +sul | su + il +sullo | su + lo +sui | su + i +sugli | su + gli +sull | su + l' +sugl | su + gl' +sulla | su + la +sulle | su + le +per | through, by +tra | among +contro | against +io | I +tu | thou +lui | he +lei | she +noi | we +voi | you +loro | they +mio | my +mia | +miei | +mie | +tuo | +tua | +tuoi | thy +tue | +suo | +sua | +suoi | his, her +sue | +nostro | our +nostra | +nostri | +nostre | +vostro | your +vostra | +vostri | +vostre | +mi | me +ti | thee +ci | us, there +vi | you, there +lo | him, the +la | her, the +li | them +le | them, the +gli | to him, the +ne | from there etc +il | the +un | a +uno | a +una | a +ma | but +ed | and +se | if +perché | why, because +anche | also +come | how +dov | where (as dov') +dove | where +che | who, that +chi | who +cui | whom +non | not +più | more +quale | who, that +quanto | how much +quanti | +quanta | +quante | +quello | that +quelli | +quella | +quelle | +questo | this +questi | +questa | +queste | +si | yes +tutto | all +tutti | all + + | single letter forms: + +a | at +c | as c' for ce or ci +e | and +i | the +l | as l' +o | or + + | forms of avere, to have (not including the infinitive): + +ho +hai +ha +abbiamo +avete +hanno +abbia +abbiate +abbiano +avrò +avrai +avrà +avremo +avrete +avranno +avrei +avresti +avrebbe +avremmo +avreste +avrebbero +avevo +avevi +aveva +avevamo +avevate +avevano +ebbi +avesti +ebbe +avemmo +aveste +ebbero +avessi +avesse +avessimo +avessero +avendo +avuto +avuta +avuti +avute + + | forms of essere, to be (not including the infinitive): +sono +sei +è +siamo +siete +sia +siate +siano +sarò +sarai +sarà +saremo +sarete +saranno +sarei +saresti +sarebbe +saremmo +sareste +sarebbero +ero +eri +era +eravamo +eravate +erano +fui +fosti +fu +fummo +foste +furono +fossi +fosse +fossimo +fossero +essendo + + | forms of fare, to do (not including the infinitive, fa, fat-): +faccio +fai +facciamo +fanno +faccia +facciate +facciano +farò +farai +farà +faremo +farete +faranno +farei +faresti +farebbe +faremmo +fareste +farebbero +facevo +facevi +faceva +facevamo +facevate +facevano +feci +facesti +fece +facemmo +faceste +fecero +facessi +facesse +facessimo +facessero +facendo + + | forms of stare, to be (not including the infinitive): +sto +stai +sta +stiamo +stanno +stia +stiate +stiano +starò +starai +starà +staremo +starete +staranno +starei +staresti +starebbe +staremmo +stareste +starebbero +stavo +stavi +stava +stavamo +stavate +stavano +stetti +stesti +stette +stemmo +steste +stettero +stessi +stesse +stessimo +stessero +stando diff --git a/src/lib-fts/stopwords/stopwords_malformed.txt b/src/lib-fts/stopwords/stopwords_malformed.txt new file mode 100644 index 00000000000..5899230617a --- /dev/null +++ b/src/lib-fts/stopwords/stopwords_malformed.txt @@ -0,0 +1,2866 @@ + + + + + + + + + + + + + + + + + + + + + + + + lucene-solr/stopwords_de.txt at master · apache/lucene-solr · GitHub + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Skip to content + + + + + + + + + + + + +
+ +
+
+ + +
+
+
+ +
+
+ + + + + +

+ + /lucene-solr + + + + + + + mirrored from git://git.apache.org/lucene-solr.git + +

+ +
+
+ + + +
+
+ +
+
+ + + + + + + +
+ +
+ + + +
+ +
+ + Find file + + +
+
+ lucene-solr/solr/example/files/conf/lang/stopwords_de.txt +
+
+ + +
+ + + be4680a + + + + + +
+ + +
+ + +
+ +
+
+
+ +
+ Raw + Blame + History +
+ + + + +
+ +
+ 295 lines (243 sloc) + + 4.47 KB +
+
+ + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
| From svn.tartarus.org/snowball/trunk/website/algorithms/german/stop.txt
| This file is distributed under the BSD License.
| See http://snowball.tartarus.org/license.php
| Also see http://www.opensource.org/licenses/bsd-license.html
| - Encoding was converted to UTF-8.
| - This notice was added.
|
| NOTE: To use this file with StopFilterFactory, you must specify format="snowball"
+
| A German stop word list. Comments begin with vertical bar. Each stop
| word is at the start of a line.
+
| The number of forms in this list is reduced significantly by passing it
| through the German stemmer.
+
+
aber | but
+
alle | all
allem
allen
aller
alles
+
als | than, as
also | so
am | an + dem
an | at
+
ander | other
andere
anderem
anderen
anderer
anderes
anderm
andern
anderr
anders
+
auch | also
auf | on
aus | out of
bei | by
bin | am
bis | until
bist | art
da | there
damit | with it
dann | then
+
der | the
den
des
dem
die
das
+
daß | that
+
derselbe | the same
derselben
denselben
desselben
demselben
dieselbe
dieselben
dasselbe
+
dazu | to that
+
dein | thy
deine
deinem
deinen
deiner
deines
+
denn | because
+
derer | of those
dessen | of him
+
dich | thee
dir | to thee
du | thou
+
dies | this
diese
diesem
diesen
dieser
dieses
+
+
doch | (several meanings)
dort | (over) there
+
+
durch | through
+
ein | a
eine
einem
einen
einer
eines
+
einig | some
einige
einigem
einigen
einiger
einiges
+
einmal | once
+
er | he
ihn | him
ihm | to him
+
es | it
etwas | something
+
euer | your
eure
eurem
euren
eurer
eures
+
für | for
gegen | towards
gewesen | p.p. of sein
hab | have
habe | have
haben | have
hat | has
hatte | had
hatten | had
hier | here
hin | there
hinter | behind
+
ich | I
mich | me
mir | to me
+
+
ihr | you, to her
ihre
ihrem
ihren
ihrer
ihres
euch | to you
+
im | in + dem
in | in
indem | while
ins | in + das
ist | is
+
jede | each, every
jedem
jeden
jeder
jedes
+
jene | that
jenem
jenen
jener
jenes
+
jetzt | now
kann | can
+
kein | no
keine
keinem
keinen
keiner
keines
+
können | can
könnte | could
machen | do
man | one
+
manche | some, many a
manchem
manchen
mancher
manches
+
mein | my
meine
meinem
meinen
meiner
meines
+
mit | with
muss | must
musste | had to
nach | to(wards)
nicht | not
nichts | nothing
noch | still, yet
nun | now
nur | only
ob | whether
oder | or
ohne | without
sehr | very
+
sein | his
seine
seinem
seinen
seiner
seines
+
selbst | self
sich | herself
+
sie | they, she
ihnen | to them
+
sind | are
so | so
+
solche | such
solchem
solchen
solcher
solches
+
soll | shall
sollte | should
sondern | but
sonst | else
über | over
um | about, around
und | and
+
uns | us
unse
unsem
unsen
unser
unses
+
unter | under
viel | much
vom | von + dem
von | from
vor | before
während | while
war | was
waren | were
warst | wast
was | what
weg | away, off
weil | because
weiter | further
+
welche | which
welchem
welchen
welcher
welches
+
wenn | when
werde | will
werden | will
wie | how
wieder | again
will | want
wir | we
wird | will
wirst | willst
wo | where
wollen | want
wollte | wanted
würde | would
würden | would
zu | to
zum | zu + dem
zur | zu + der
zwar | indeed
zwischen | between
+
+ +
+ +
+ + + + +
+
+
+ + +
+
+ +
+ +
+ +
+ + + + + + + +
+ + + Something went wrong with that request. Please try again. +
+ + + + + + + + + + + + + + diff --git a/src/lib-fts/stopwords/stopwords_nl.txt b/src/lib-fts/stopwords/stopwords_nl.txt new file mode 100644 index 00000000000..6fc4ce7e359 --- /dev/null +++ b/src/lib-fts/stopwords/stopwords_nl.txt @@ -0,0 +1,118 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/dutch/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + + | A Dutch stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + | This is a ranked list (commonest to rarest) of stopwords derived from + | a large sample of Dutch text. + + | Dutch stop words frequently exhibit homonym clashes. These are indicated + | clearly below. + +de | the +en | and +van | of, from +ik | I, the ego +te | (1) chez, at etc, (2) to, (3) too +dat | that, which +die | that, those, who, which +in | in, inside +een | a, an, one +hij | he +het | the, it +niet | not, nothing, naught +zijn | (1) to be, being, (2) his, one's, its +is | is +was | (1) was, past tense of all persons sing. of 'zijn' (to be) (2) wax, (3) the washing, (4) rise of river +op | on, upon, at, in, up, used up +aan | on, upon, to (as dative) +met | with, by +als | like, such as, when +voor | (1) before, in front of, (2) furrow +had | had, past tense all persons sing. of 'hebben' (have) +er | there +maar | but, only +om | round, about, for etc +hem | him +dan | then +zou | should/would, past tense all persons sing. of 'zullen' +of | or, whether, if +wat | what, something, anything +mijn | possessive and noun 'mine' +men | people, 'one' +dit | this +zo | so, thus, in this way +door | through by +over | over, across +ze | she, her, they, them +zich | oneself +bij | (1) a bee, (2) by, near, at +ook | also, too +tot | till, until +je | you +mij | me +uit | out of, from +der | Old Dutch form of 'van der' still found in surnames +daar | (1) there, (2) because +haar | (1) her, their, them, (2) hair +naar | (1) unpleasant, unwell etc, (2) towards, (3) as +heb | present first person sing. of 'to have' +hoe | how, why +heeft | present third person sing. of 'to have' +hebben | 'to have' and various parts thereof +deze | this +u | you +want | (1) for, (2) mitten, (3) rigging +nog | yet, still +zal | 'shall', first and third person sing. of verb 'zullen' (will) +me | me +zij | she, they +nu | now +ge | 'thou', still used in Belgium and south Netherlands +geen | none +omdat | because +iets | something, somewhat +worden | to become, grow, get +toch | yet, still +al | all, every, each +waren | (1) 'were' (2) to wander, (3) wares, (3) +veel | much, many +meer | (1) more, (2) lake +doen | to do, to make +toen | then, when +moet | noun 'spot/mote' and present form of 'to must' +ben | (1) am, (2) 'are' in interrogative second person singular of 'to be' +zonder | without +kan | noun 'can' and present form of 'to be able' +hun | their, them +dus | so, consequently +alles | all, everything, anything +onder | under, beneath +ja | yes, of course +eens | once, one day +hier | here +wie | who +werd | imperfect third person sing. of 'become' +altijd | always +doch | yet, but etc +wordt | present third person sing. of 'become' +wezen | (1) to be, (2) 'been' as in 'been fishing', (3) orphans +kunnen | to be able +ons | us/our +zelf | self +tegen | against, towards, at +na | after, near +reeds | already +wil | (1) present tense of 'want', (2) 'will', noun, (3) fender +kon | could; past tense of 'to be able' +niets | nothing +uw | your +iemand | somebody +geweest | been; past participle of 'be' +andere | other diff --git a/src/lib-fts/stopwords_no.txt b/src/lib-fts/stopwords/stopwords_no.txt similarity index 100% rename from src/lib-fts/stopwords_no.txt rename to src/lib-fts/stopwords/stopwords_no.txt diff --git a/src/lib-fts/stopwords/stopwords_pt.txt b/src/lib-fts/stopwords/stopwords_pt.txt new file mode 100644 index 00000000000..37449615ef6 --- /dev/null +++ b/src/lib-fts/stopwords/stopwords_pt.txt @@ -0,0 +1,252 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/portuguese/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + + | A Portuguese stop word list. Comments begin with vertical bar. Each stop + | word is at the start of a line. + + + | The following is a ranked list (commonest to rarest) of stopwords + | deriving from a large sample of text. + + | Extra words have been added at the end. + +de | of, from +a | the; to, at; her +o | the; him +que | who, that +e | and +do | de + o +da | de + a +em | in +um | a +para | for + | é from SER +com | with +não | not, no +uma | a +os | the; them +no | em + o +se | himself etc +na | em + a +por | for +mais | more +as | the; them +dos | de + os +como | as, like +mas | but + | foi from SER +ao | a + o +ele | he +das | de + as + | tem from TER +à | a + a +seu | his +sua | her +ou | or + | ser from SER +quando | when +muito | much + | há from HAV +nos | em + os; us +já | already, now + | está from EST +eu | I +também | also +só | only, just +pelo | per + o +pela | per + a +até | up to +isso | that +ela | he +entre | between + | era from SER +depois | after +sem | without +mesmo | same +aos | a + os + | ter from TER +seus | his +quem | whom +nas | em + as +me | me +esse | that +eles | they + | estão from EST +você | you + | tinha from TER + | foram from SER +essa | that +num | em + um +nem | nor +suas | her +meu | my +às | a + as +minha | my + | têm from TER +numa | em + uma +pelos | per + os +elas | they + | havia from HAV + | seja from SER +qual | which + | será from SER +nós | we + | tenho from TER +lhe | to him, her +deles | of them +essas | those +esses | those +pelas | per + as +este | this + | fosse from SER +dele | of him + + | other words. There are many contractions such as naquele = em+aquele, + | mo = me+o, but they are rare. + | Indefinite article plural forms are also rare. + +tu | thou +te | thee +vocês | you (plural) +vos | you +lhes | to them +meus | my +minhas +teu | thy +tua +teus +tuas +nosso | our +nossa +nossos +nossas + +dela | of her +delas | of them + +esta | this +estes | these +estas | these +aquele | that +aquela | that +aqueles | those +aquelas | those +isto | this +aquilo | that + + | forms of estar, to be (not including the infinitive): +estou +está +estamos +estão +estive +esteve +estivemos +estiveram +estava +estávamos +estavam +estivera +estivéramos +esteja +estejamos +estejam +estivesse +estivéssemos +estivessem +estiver +estivermos +estiverem + + | forms of haver, to have (not including the infinitive): +hei +há +havemos +hão +houve +houvemos +houveram +houvera +houvéramos +haja +hajamos +hajam +houvesse +houvéssemos +houvessem +houver +houvermos +houverem +houverei +houverá +houveremos +houverão +houveria +houveríamos +houveriam + + | forms of ser, to be (not including the infinitive): +sou +somos +são +era +éramos +eram +fui +foi +fomos +foram +fora +fôramos +seja +sejamos +sejam +fosse +fôssemos +fossem +for +formos +forem +serei +será +seremos +serão +seria +seríamos +seriam + + | forms of ter, to have (not including the infinitive): +tenho +tem +temos +tém +tinha +tínhamos +tinham +tive +teve +tivemos +tiveram +tivera +tivéramos +tenha +tenhamos +tenham +tivesse +tivéssemos +tivessem +tiver +tivermos +tiverem +terei +terá +teremos +terão +teria +teríamos +teriam diff --git a/src/lib-fts/stopwords/stopwords_ro.txt b/src/lib-fts/stopwords/stopwords_ro.txt new file mode 100644 index 00000000000..4fdee90a5ba --- /dev/null +++ b/src/lib-fts/stopwords/stopwords_ro.txt @@ -0,0 +1,233 @@ +# This file was created by Jacques Savoy and is distributed under the BSD license. +# See http://members.unine.ch/jacques.savoy/clef/index.html. +# Also see http://www.opensource.org/licenses/bsd-license.html +acea +aceasta +această +aceea +acei +aceia +acel +acela +acele +acelea +acest +acesta +aceste +acestea +aceşti +aceştia +acolo +acum +ai +aia +aibă +aici +al +ăla +ale +alea +ălea +altceva +altcineva +am +ar +are +aş +aşadar +asemenea +asta +ăsta +astăzi +astea +ăstea +ăştia +asupra +aţi +au +avea +avem +aveţi +azi +bine +bucur +bună +ca +că +căci +când +care +cărei +căror +cărui +cât +câte +câţi +către +câtva +ce +cel +ceva +chiar +cînd +cine +cineva +cît +cîte +cîţi +cîtva +contra +cu +cum +cumva +curând +curînd +da +dă +dacă +dar +datorită +de +deci +deja +deoarece +departe +deşi +din +dinaintea +dintr +dintre +drept +după +ea +ei +el +ele +eram +este +eşti +eu +face +fără +fi +fie +fiecare +fii +fim +fiţi +iar +ieri +îi +îl +îmi +împotriva +în +înainte +înaintea +încât +încît +încotro +între +întrucât +întrucît +îţi +la +lângă +le +li +lîngă +lor +lui +mă +mâine +mea +mei +mele +mereu +meu +mi +mine +mult +multă +mulţi +ne +nicăieri +nici +nimeni +nişte +noastră +noastre +noi +noştri +nostru +nu +ori +oricând +oricare +oricât +orice +oricînd +oricine +oricît +oricum +oriunde +până +pe +pentru +peste +pînă +poate +pot +prea +prima +primul +prin +printr +sa +să +săi +sale +sau +său +se +şi +sînt +sîntem +sînteţi +spre +sub +sunt +suntem +sunteţi +ta +tăi +tale +tău +te +ţi +ţie +tine +toată +toate +tot +toţi +totuşi +tu +un +una +unde +undeva +unei +unele +uneori +unor +vă +vi +voastră +voastre +voi +voştri +vostru +vouă +vreo +vreun diff --git a/src/lib-fts/stopwords/stopwords_ru.txt b/src/lib-fts/stopwords/stopwords_ru.txt new file mode 100644 index 00000000000..36b0d9fd277 --- /dev/null +++ b/src/lib-fts/stopwords/stopwords_ru.txt @@ -0,0 +1,242 @@ + | From svn.tartarus.org/snowball/trunk/website/algorithms/russian/stop.txt + | This file is distributed under the BSD License. + | See http://snowball.tartarus.org/license.php + | Also see http://www.opensource.org/licenses/bsd-license.html + | - Encoding was converted to UTF-8. + | - This notice was added. + | + + | a russian stop word list. comments begin with vertical bar. each stop + | word is at the start of a line. + + | this is a ranked list (commonest to rarest) of stopwords derived from + | a large text sample. + + | letter `ё' is translated to `е'. + +и | and +в | in/into +во | alternative form +не | not +что | what/that +он | he +на | on/onto +я | i +с | from +со | alternative form +как | how +а | milder form of `no' (but) +то | conjunction and form of `that' +все | all +она | she +так | so, thus +его | him +но | but +да | yes/and +ты | thou +к | towards, by +у | around, chez +же | intensifier particle +вы | you +за | beyond, behind +бы | conditional/subj. particle +по | up to, along +только | only +ее | her +мне | to me +было | it was +вот | here is/are, particle +от | away from +меня | me +еще | still, yet, more +нет | no, there isnt/arent +о | about +из | out of +ему | to him +теперь | now +когда | when +даже | even +ну | so, well +вдруг | suddenly +ли | interrogative particle +если | if +уже | already, but homonym of `narrower' +или | or +ни | neither +быть | to be +был | he was +него | prepositional form of его +до | up to +вас | you accusative +нибудь | indef. suffix preceded by hyphen +опять | again +уж | already, but homonym of `adder' +вам | to you +сказал | he said +ведь | particle `after all' +там | there +потом | then +себя | oneself +ничего | nothing +ей | to her +может | usually with `быть' as `maybe' +они | they +тут | here +где | where +есть | there is/are +надо | got to, must +ней | prepositional form of ей +для | for +мы | we +тебя | thee +их | them, their +чем | than +была | she was +сам | self +чтоб | in order to +без | without +будто | as if +человек | man, person, one +чего | genitive form of `what' +раз | once +тоже | also +себе | to oneself +под | beneath +жизнь | life +будет | will be +ж | short form of intensifer particle `же' +тогда | then +кто | who +этот | this +говорил | was saying +того | genitive form of `that' +потому | for that reason +этого | genitive form of `this' +какой | which +совсем | altogether +ним | prepositional form of `его', `они' +здесь | here +этом | prepositional form of `этот' +один | one +почти | almost +мой | my +тем | instrumental/dative plural of `тот', `то' +чтобы | full form of `in order that' +нее | her (acc.) +кажется | it seems +сейчас | now +были | they were +куда | where to +зачем | why +сказать | to say +всех | all (acc., gen. preposn. plural) +никогда | never +сегодня | today +можно | possible, one can +при | by +наконец | finally +два | two +об | alternative form of `о', about +другой | another +хоть | even +после | after +над | above +больше | more +тот | that one (masc.) +через | across, in +эти | these +нас | us +про | about +всего | in all, only, of all +них | prepositional form of `они' (they) +какая | which, feminine +много | lots +разве | interrogative particle +сказала | she said +три | three +эту | this, acc. fem. sing. +моя | my, feminine +впрочем | moreover, besides +хорошо | good +свою | ones own, acc. fem. sing. +этой | oblique form of `эта', fem. `this' +перед | in front of +иногда | sometimes +лучше | better +чуть | a little +том | preposn. form of `that one' +нельзя | one must not +такой | such a one +им | to them +более | more +всегда | always +конечно | of course +всю | acc. fem. sing of `all' +между | between + + + | b: some paradigms + | + | personal pronouns + | + | я меня мне мной [мною] + | ты тебя тебе тобой [тобою] + | он его ему им [него, нему, ним] + | она ее эи ею [нее, нэи, нею] + | оно его ему им [него, нему, ним] + | + | мы нас нам нами + | вы вас вам вами + | они их им ими [них, ним, ними] + | + | себя себе собой [собою] + | + | demonstrative pronouns: этот (this), тот (that) + | + | этот эта это эти + | этого эты это эти + | этого этой этого этих + | этому этой этому этим + | этим этой этим [этою] этими + | этом этой этом этих + | + | тот та то те + | того ту то те + | того той того тех + | тому той тому тем + | тем той тем [тою] теми + | том той том тех + | + | determinative pronouns + | + | (a) весь (all) + | + | весь вся все все + | всего всю все все + | всего всей всего всех + | всему всей всему всем + | всем всей всем [всею] всеми + | всем всей всем всех + | + | (b) сам (himself etc) + | + | сам сама само сами + | самого саму само самих + | самого самой самого самих + | самому самой самому самим + | самим самой самим [самою] самими + | самом самой самом самих + | + | stems of verbs `to be', `to have', `to do' and modal + | + | быть бы буд быв есть суть + | име + | дел + | мог мож мочь + | уме + | хоч хот + | долж + | можн + | нужн + | нельзя + diff --git a/src/lib-fts/stopwords_sv.txt b/src/lib-fts/stopwords/stopwords_sv.txt similarity index 100% rename from src/lib-fts/stopwords_sv.txt rename to src/lib-fts/stopwords/stopwords_sv.txt diff --git a/src/lib-fts/stopwords_fi.txt b/src/lib-fts/stopwords_fi.txt deleted file mode 100644 index addad798c4b..00000000000 --- a/src/lib-fts/stopwords_fi.txt +++ /dev/null @@ -1,95 +0,0 @@ - | From svn.tartarus.org/snowball/trunk/website/algorithms/finnish/stop.txt - | This file is distributed under the BSD License. - | See http://snowball.tartarus.org/license.php - | Also see http://www.opensource.org/licenses/bsd-license.html - | - Encoding was converted to UTF-8. - | - This notice was added. - -| forms of BE - -olla -olen -olet -on -olemme -olette -ovat -ole | negative form - -oli -olisi -olisit -olisin -olisimme -olisitte -olisivat -olit -olin -olimme -olitte -olivat -ollut -olleet - -en | negation -et -ei -emme -ette -eivät - -|Nom Gen Acc Part Iness Elat Illat Adess Ablat Allat Ess Trans -minä minun minut minua minussa minusta minuun minulla minulta minulle | I -sinä sinun sinut sinua sinussa sinusta sinuun sinulla sinulta sinulle | you -hän hänen hänet häntä hänessä hänestä häneen hänellä häneltä hänelle | he she -me meidän meidät meitä meissä meistä meihin meillä meiltä meille | we -te teidän teidät teitä teissä teistä teihin teillä teiltä teille | you -he heidän heidät heitä heissä heistä heihin heillä heiltä heille | they - -tämä tämän tätä tässä tästä tähän tallä tältä tälle tänä täksi | this -tuo tuon tuotä tuossa tuosta tuohon tuolla tuolta tuolle tuona tuoksi | that -se sen sitä siinä siitä siihen sillä siltä sille sinä siksi | it -nämä näiden näitä näissä näistä näihin näillä näiltä näille näinä näiksi | these -nuo noiden noita noissa noista noihin noilla noilta noille noina noiksi | those -ne niiden niitä niissä niistä niihin niillä niiltä niille niinä niiksi | they - -kuka kenen kenet ketä kenessä kenestä keneen kenellä keneltä kenelle kenenä keneksi| who -ketkä keiden ketkä keitä keissä keistä keihin keillä keiltä keille keinä keiksi | (pl) -mikä minkä minkä mitä missä mistä mihin millä miltä mille minä miksi | which what -mitkä | (pl) - -joka jonka jota jossa josta johon jolla jolta jolle jona joksi | who which -jotka joiden joita joissa joista joihin joilla joilta joille joina joiksi | (pl) - -| conjunctions - -että | that -ja | and -jos | if -koska | because -kuin | than -mutta | but -niin | so -sekä | and -sillä | for -tai | or -vaan | but -vai | or -vaikka | although - - -| prepositions - -kanssa | with -mukaan | according to -noin | about -poikki | across -yli | over, across - -| other - -kun | when -niin | so -nyt | now -itse | self - diff --git a/src/lib-fts/test-fts-filter.c b/src/lib-fts/test-fts-filter.c index 55dd25f640e..6c5f1a7b2c8 100644 --- a/src/lib-fts/test-fts-filter.c +++ b/src/lib-fts/test-fts-filter.c @@ -348,6 +348,22 @@ static void test_fts_filter_stopwords_fail_lazy_init(void) } +static void test_fts_filter_stopwords_malformed(void) +{ + const struct fts_language malformed = { .name = "malformed" }; + struct fts_filter *filter = NULL; + const char *error = NULL, *token = "foobar"; + + test_begin("fts filter stopwords, malformed list"); + test_assert(fts_filter_create(fts_filter_stopwords, NULL, &malformed, stopword_settings, &filter, &error) == 0); + test_expect_error_string("seems empty. Is the file correctly formatted?"); + test_assert(fts_filter_filter(filter, &token, &error) > 0); + test_expect_no_more_errors(); + fts_filter_unref(&filter); + test_end(); + +} + #ifdef HAVE_FTS_STEMMER static void test_fts_filter_stemmer_snowball_stem_english(void) { @@ -539,20 +555,20 @@ static void test_fts_filter_normalizer_french(void) FILE *input; const char * const settings[] = {"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove", NULL}; - char buf[4096] = {0}; + char buf[250] = {0}; const char *error = NULL; const char *tokens; unsigned char sha512_digest[SHA512_RESULTLEN]; struct sha512_ctx ctx; const unsigned char correct_digest[] = { - 0x78, 0x1e, 0xb9, 0x04, 0xa4, 0x92, 0xca, 0x88, - 0x1e, 0xef, 0x7b, 0xc8, 0x3e, 0x4a, 0xa8, 0xdb, - 0x9c, 0xd4, 0x42, 0x5c, 0x64, 0x81, 0x06, 0xd5, - 0x72, 0x93, 0x38, 0x0c, 0x09, 0xce, 0xbe, 0xdf, - 0x65, 0xff, 0x36, 0x35, 0x05, 0x77, 0xcc, 0xc6, - 0xff, 0x44, 0x2c, 0x31, 0x10, 0x00, 0xf6, 0x8d, - 0x15, 0x25, 0x1e, 0x54, 0x67, 0x2a, 0x5b, 0xc1, - 0xdb, 0x84, 0xc5, 0x0d, 0x43, 0x7e, 0x8c, 0x70}; + 0x06, 0x80, 0xf1, 0x81, 0xf2, 0xed, 0xfb, 0x6d, + 0xcd, 0x7d, 0xcb, 0xbd, 0xc4, 0x87, 0xc3, 0xf6, + 0xb8, 0x6a, 0x01, 0x82, 0xdf, 0x0a, 0xb5, 0x92, + 0x6b, 0x9b, 0x7b, 0x21, 0x5e, 0x62, 0x40, 0xbd, + 0xbf, 0x15, 0xb9, 0x7b, 0x75, 0x9c, 0x4e, 0xc9, + 0xe8, 0x48, 0xaa, 0x08, 0x63, 0xf2, 0xa0, 0x6c, + 0x20, 0x4c, 0x01, 0xe3, 0xb3, 0x4f, 0x15, 0xc6, + 0x8c, 0xd6, 0x7a, 0xb7, 0xc5, 0xc6, 0x85, 0x00}; const char *udhr_path; test_begin("fts filter normalizer French UDHR"); @@ -650,6 +666,39 @@ static void test_fts_filter_normalizer_invalid_id(void) test_end(); } +static void test_fts_filter_normalizer_oversized(void) +{ + struct fts_filter *norm = NULL; + const char *settings[] = + {"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove", "maxlen", "250", + NULL}; + const char *error = NULL; + const char *token = "\xe4\x95\x91\x25\xe2\x94\xad\xe1\x90\xad\xee\x94\x81\xe2\x8e\x9e" + "\xe7\x9a\xb7\xea\xbf\x97\xe3\xb2\x8f\xe4\x9c\xbe\xee\xb4\x98\xe1" + "\x8d\x99\xe2\x91\x83\xe3\xb1\xb8\xef\xbf\xbd\xe8\xbb\x9c\xef\xbf" + "\xbd\xea\xbb\x98\xea\xb5\xac\xe4\x87\xae\xe4\x88\x93\xe9\x86\x8f" + "\xe9\x86\x83\xe6\x8f\x8d\xe7\xa3\x9d\xed\x89\x96\xe2\x89\x85\xe6" + "\x8c\x82\xec\x80\x98\xee\x91\x96\xe7\xa8\x8a\xec\xbc\x85\xeb\x9c" + "\xbd\xeb\x97\x95\xe3\xa4\x9d\xd7\xb1\xea\xa7\x94\xe0\xbb\xac\xee" + "\x95\x87\xd5\x9d\xe8\xba\x87\xee\x8b\xae\xe5\xb8\x80\xe9\x8d\x82" + "\xe7\xb6\x8c\xe7\x9b\xa0\xef\x82\x9f\xed\x96\xa4\xe3\x8d\xbc\xe1" + "\x81\xbd\xe9\x81\xb2\xea\xac\xac\xec\x9b\x98\xe7\x84\xb2\xee\xaf" + "\xbc\xeb\xa2\x9d\xe9\x86\xb3\xe0\xb0\x89\xeb\x80\xb6\xe3\x8c\x9d" + "\xe9\x8f\x9e\xe2\xae\x8a\xee\x9e\x9a\xef\xbf\xbd\xe7\xa3\x9b\xe4" + "\xa3\x8b\xe4\x82\xb9\xeb\x8e\x93\xec\xb5\x82\xe5\xa7\x81\xe2\x8c" + "\x97\xea\xbb\xb4\xe5\x85\xb7\xeb\x96\xbe\xe7\x97\x91\xea\xbb\x98" + "\xe6\xae\xb4\xe9\x8a\x85\xc4\xb9\xe4\x90\xb2\xe9\x96\xad\xef\x90" + "\x9c\xe5\xa6\xae\xe9\x93\x91\xe8\x87\xa1"; + + test_begin("fts filter normalizer over-sized token"); + test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0); + test_assert(error == NULL); + test_assert(fts_filter_filter(norm, &token, &error) >= 0); + test_assert(strlen(token) <= 250); + fts_filter_unref(&norm); + test_end(); +} + #ifdef HAVE_FTS_STEMMER static void test_fts_filter_normalizer_stopwords_stemmer_eng(void) { @@ -893,6 +942,7 @@ int main(void) test_fts_filter_stopwords_fra, test_fts_filter_stopwords_no, test_fts_filter_stopwords_fail_lazy_init, + test_fts_filter_stopwords_malformed, #ifdef HAVE_FTS_STEMMER test_fts_filter_stemmer_snowball_stem_english, test_fts_filter_stemmer_snowball_stem_french, @@ -905,6 +955,7 @@ int main(void) test_fts_filter_normalizer_empty, test_fts_filter_normalizer_baddata, test_fts_filter_normalizer_invalid_id, + test_fts_filter_normalizer_oversized, #ifdef HAVE_FTS_STEMMER test_fts_filter_normalizer_stopwords_stemmer_eng, test_fts_filter_stopwords_normalizer_stemmer_no, diff --git a/src/lib-fts/test-fts-icu.c b/src/lib-fts/test-fts-icu.c index 6eab62015af..f1748bd63cb 100644 --- a/src/lib-fts/test-fts-icu.c +++ b/src/lib-fts/test-fts-icu.c @@ -11,18 +11,17 @@ static void test_fts_icu_utf8_to_utf16_ascii_resize(void) { - buffer_t *dest = buffer_create_dynamic(pool_datastack_create(), 5); + buffer_t *dest = buffer_create_dynamic(pool_datastack_create(), 4); test_begin("fts_icu_utf8_to_utf16 ascii resize"); - /* dynamic buffers reserve +1 for str_c()'s NUL, so 5 -> 4 */ - test_assert(buffer_get_size(dest) == 5); + test_assert(buffer_get_writable_size(dest) == 4); fts_icu_utf8_to_utf16(dest, "12"); test_assert(dest->used == 4); - test_assert(buffer_get_size(dest) == 5); + test_assert(buffer_get_writable_size(dest) == 4); fts_icu_utf8_to_utf16(dest, "123"); test_assert(dest->used == 6); - test_assert(buffer_get_size(dest) == 8); + test_assert(buffer_get_writable_size(dest) == 7); fts_icu_utf8_to_utf16(dest, "12345"); test_assert(dest->used == 10); @@ -38,7 +37,7 @@ static void test_fts_icu_utf8_to_utf16_32bit_resize(void) test_begin("fts_icu_utf8_to_utf16 32bit resize"); for (i = 2; i <= 5; i++) { dest = buffer_create_dynamic(pool_datastack_create(), i); - test_assert(buffer_get_size(dest) == i); + test_assert(buffer_get_writable_size(dest) == i); fts_icu_utf8_to_utf16(dest, "\xF0\x90\x90\x80"); /* 0x10400 */ test_assert(dest->used == 4); } @@ -69,7 +68,7 @@ static void test_fts_icu_utf16_to_utf8_resize(void) test_begin("fts_icu_utf16_to_utf8 resize"); for (i = 2; i <= 6; i++) { dest = t_str_new(i); - test_assert(buffer_get_size(dest) == i); + test_assert(buffer_get_writable_size(dest) == i); fts_icu_utf16_to_utf8(dest, &src, 1); test_assert(dest->used == 3); test_assert(strcmp(str_c(dest), UNICODE_REPLACEMENT_CHAR_UTF8) == 0); @@ -131,7 +130,7 @@ static void test_fts_icu_translate_resize(void) buffer_set_used_size(src_utf16, 0); fts_icu_utf8_to_utf16(src_utf16, src_utf8); dest = buffer_create_dynamic(pool_datastack_create(), i); - test_assert(buffer_get_size(dest) == i); + test_assert(buffer_get_writable_size(dest) == i); test_assert(fts_icu_translate(dest, src_utf16->data, src_utf16->used/sizeof(UChar), translit, &error) == 0); @@ -159,12 +158,12 @@ static void test_fts_icu_lcase_resize(void) unsigned int i; test_begin("fts_icu_lcase resize"); - for (i = 2; i <= 4; i++) { + for (i = 1; i <= 3; i++) { dest = t_str_new(i); - test_assert(buffer_get_size(dest) == i); + test_assert(buffer_get_writable_size(dest) == i); fts_icu_lcase(dest, src); test_assert(strcmp(str_c(dest), "a\xC3\xA4") == 0); - test_assert(buffer_get_size(dest) == 4); + test_assert(buffer_get_writable_size(dest) == 3); } test_end(); diff --git a/src/lib-fts/test-fts-tokenizer.c b/src/lib-fts/test-fts-tokenizer.c index 06e14ed8f83..06fa79041c7 100644 --- a/src/lib-fts/test-fts-tokenizer.c +++ b/src/lib-fts/test-fts-tokenizer.c @@ -4,16 +4,22 @@ #include "unichar.h" #include "test-common.h" #include "fts-tokenizer.h" +#include "fts-tokenizer-common.h" #include "fts-tokenizer-private.h" #include "fts-tokenizer-generic-private.h" - +/*there should be a trailing space ' ' at the end of each string except the last one*/ #define TEST_INPUT_ADDRESS \ "@invalid invalid@ Abc Dfg , " \ "Bar Baz " \ "Foo Bar (comment)foo.bar@host.example.org " \ "foo, foo@domain " \ - "abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.tld" + "abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.tld " \ + "trailing, period@blue.com. " \ + "multi-trialing, mul@trail.com..... " \ + "m@s " \ + "hypen@hypen-hypen.com " \ + "hypen@hypen-hypen-sick.com.-" static const char *test_inputs[] = { /* generic things and word truncation: */ @@ -78,11 +84,11 @@ test_tokenizer_inputoutput(struct fts_tokenizer *tok, const char *_input, /* test all input at once */ outi = first_outi; while (fts_tokenizer_next(tok, input, input_len, &token, &error) > 0) { - test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); + test_assert_strcmp(token, expected_output[outi]); outi++; } while (fts_tokenizer_next(tok, NULL, 0, &token, &error) > 0) { - test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); + test_assert_strcmp(token, expected_output[outi]); outi++; } test_assert_idx(expected_output[outi] == NULL, outi); @@ -92,12 +98,12 @@ test_tokenizer_inputoutput(struct fts_tokenizer *tok, const char *_input, for (i = 0; i < input_len; i += char_len) { char_len = uni_utf8_char_bytes(input[i]); while (fts_tokenizer_next(tok, input+i, char_len, &token, &error) > 0) { - test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); + test_assert_strcmp(token, expected_output[outi]); outi++; } } while (fts_tokenizer_final(tok, &token, &error) > 0) { - test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); + test_assert_strcmp(token, expected_output[outi]); outi++; } test_assert_idx(expected_output[outi] == NULL, outi); @@ -109,12 +115,12 @@ test_tokenizer_inputoutput(struct fts_tokenizer *tok, const char *_input, for (char_len = 0; char_len < max; ) char_len += uni_utf8_char_bytes(input[i+char_len]); while (fts_tokenizer_next(tok, input+i, char_len, &token, &error) > 0) { - test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); + test_assert_strcmp(token, expected_output[outi]); outi++; } } while (fts_tokenizer_final(tok, &token, &error) > 0) { - test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi); + test_assert_strcmp(token, expected_output[outi]); outi++; } test_assert_idx(expected_output[outi] == NULL, outi); @@ -309,7 +315,13 @@ static void test_fts_tokenizer_address_only(void) static const char *const expected_output[] = { "abc.dfg@example.com", "bar@example.org", "foo.bar@host.example.org", "foo@domain", - "abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstu", NULL + "abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstu", + "period@blue.com", /*trailing period '.' in email */ + "mul@trail.com", + "m@s", /*one letter local-part and domain name */ + "hypen@hypen-hypen.com", + "hypen@hypen-hypen-sick.com", + NULL }; struct fts_tokenizer *tok; const char *error; @@ -328,7 +340,13 @@ static void test_fts_tokenizer_address_parent(const char *name, const char * con "invalid", "invalid", "Abc", "Dfg", "abc", "dfg", "example", "com", "abc.dfg@example.com", "Bar", "Baz", "bar", "example", "org", "bar@example.org", "Foo", "Bar", "comment", "foo", "bar", "host", "example", "org", "foo.bar@host.example.org", - "foo", "foo", "domain", "foo@domain", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyzabcde", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz","tld", "abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstu", NULL + "foo", "foo", "domain", "foo@domain", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyzabcde", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz","tld", "abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstu", + "trailing", "period", "blue", "com", "period@blue.com", + "multi", "trialing", "mul", "trail", "com", "mul@trail.com", + "m", "s", "m@s", + "hypen", "hypen", "hypen", "com", "hypen@hypen-hypen.com", + "hypen", "hypen", "hypen", "sick", "com", "hypen@hypen-hypen-sick.com", + NULL }; struct fts_tokenizer *tok, *gen_tok; const char *error; @@ -360,7 +378,13 @@ static void test_fts_tokenizer_address_search(void) "invalid", "invalid", "Abc", "Dfg", "abc.dfg@example.com", "Bar", "Baz", "bar@example.org", "Foo", "Bar", "comment", "foo.bar@host.example.org", - "foo", "foo@domain", "abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstu", NULL + "foo", "foo@domain", "abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstu", + "trailing", "period@blue.com", + "multi", "trialing", "mul@trail.com", + "m@s", + "hypen@hypen-hypen.com", + "hypen@hypen-hypen-sick.com", + NULL }; static const char *const settings[] = { "search", "", NULL }; struct fts_tokenizer *tok, *gen_tok; @@ -400,6 +424,18 @@ static void test_fts_tokenizer_address_search(void) test_end(); } +static void test_fts_tokenizer_delete_trailing_partial_char() +{ + const char* str[] = {"\x7f", "\xC2\x80", "\xE0\x80\x80","\xF0\x80\x80\x80"}; + unsigned int i; + size_t new_size; + for (i = 0; i < 4; i++) { + new_size = i+1; + fts_tokenizer_delete_trailing_partial_char((unsigned char*)str[i], &new_size); + test_assert( i+1 == new_size); + } +} + int main(void) { static void (*test_functions[])(void) = { @@ -411,6 +447,7 @@ int main(void) test_fts_tokenizer_address_parent_simple, test_fts_tokenizer_address_parent_tr29, test_fts_tokenizer_address_search, + test_fts_tokenizer_delete_trailing_partial_char, NULL }; int ret; @@ -418,5 +455,6 @@ int main(void) fts_tokenizers_init(); ret = test_run(test_functions); fts_tokenizers_deinit(); + return ret; } diff --git a/src/lib-http/Makefile.am b/src/lib-http/Makefile.am index 41d93816e35..967002e7b81 100644 --- a/src/lib-http/Makefile.am +++ b/src/lib-http/Makefile.am @@ -64,7 +64,9 @@ test_programs = \ test_nocheck_programs = \ test-http-payload \ test-http-client \ - test-http-server + test-http-client-errors \ + test-http-server \ + test-http-server-errors noinst_PROGRAMS = $(test_programs) $(test_nocheck_programs) @@ -131,16 +133,14 @@ test_http_request_parser_LDADD = \ $(test_libs) test_http_request_parser_DEPENDENCIES = $(test_deps) -test_http_payload_SOURCES = test-http-payload.c -test_http_payload_LDFLAGS = -export-dynamic -test_http_payload_LDADD = \ +test_http_libs = \ libhttp.la \ ../lib-dns/libdns.la \ ../lib-ssl-iostream/libssl_iostream.la \ ../lib-master/libmaster.la \ ../lib-settings/libsettings.la \ $(test_libs) -test_http_payload_DEPENDENCIES = \ +test_http_deps = \ libhttp.la \ ../lib-dns/libdns.la \ ../lib-ssl-iostream/libssl_iostream.la \ @@ -148,39 +148,40 @@ test_http_payload_DEPENDENCIES = \ ../lib-settings/libsettings.la \ $(test_deps) +test_http_payload_SOURCES = test-http-payload.c +test_http_payload_LDFLAGS = -export-dynamic +test_http_payload_LDADD = \ + $(test_http_libs) +test_http_payload_DEPENDENCIES = \ + $(test_http_deps) + test_http_client_SOURCES = test-http-client.c test_http_client_LDFLAGS = -export-dynamic test_http_client_LDADD = \ - libhttp.la \ - ../lib-dns/libdns.la \ - ../lib-ssl-iostream/libssl_iostream.la \ - ../lib-master/libmaster.la \ - ../lib-settings/libsettings.la \ - $(test_libs) + $(test_http_libs) test_http_client_DEPENDENCIES = \ - libhttp.la \ - ../lib-dns/libdns.la \ - ../lib-ssl-iostream/libssl_iostream.la \ - ../lib-master/libmaster.la \ - ../lib-settings/libsettings.la \ - $(test_deps) + $(test_http_deps) + +test_http_client_errors_SOURCES = test-http-client-errors.c +test_http_client_errors_LDFLAGS = -export-dynamic +test_http_client_errors_LDADD = \ + $(test_http_libs) +test_http_client_errors_DEPENDENCIES = \ + $(test_http_deps) test_http_server_SOURCES = test-http-server.c test_http_server_LDFLAGS = -export-dynamic test_http_server_LDADD = \ - libhttp.la \ - ../lib-dns/libdns.la \ - ../lib-ssl-iostream/libssl_iostream.la \ - ../lib-master/libmaster.la \ - ../lib-settings/libsettings.la \ - $(test_libs) + $(test_http_libs) test_http_server_DEPENDENCIES = \ - libhttp.la \ - ../lib-dns/libdns.la \ - ../lib-ssl-iostream/libssl_iostream.la \ - ../lib-master/libmaster.la \ - ../lib-settings/libsettings.la \ - $(test_deps) + $(test_http_deps) + +test_http_server_errors_SOURCES = test-http-server-errors.c +test_http_server_errors_LDFLAGS = -export-dynamic +test_http_server_errors_LDADD = \ + $(test_http_libs) +test_http_server_errors_DEPENDENCIES = \ + $(test_http_deps) check: check-am check-test check-test: all-am diff --git a/src/lib-http/http-client-connection.c b/src/lib-http/http-client-connection.c index f725e5452b0..567a0df3756 100644 --- a/src/lib-http/http-client-connection.c +++ b/src/lib-http/http-client-connection.c @@ -46,8 +46,6 @@ http_client_connection_debug(struct http_client_connection *conn, static void http_client_connection_ready(struct http_client_connection *conn); static void http_client_connection_input(struct connection *_conn); -static void -http_client_connection_disconnect(struct http_client_connection *conn); unsigned int http_client_connection_count_pending(struct http_client_connection *conn) @@ -68,15 +66,33 @@ static void http_client_connection_retry_requests(struct http_client_connection *conn, unsigned int status, const char *error) { - struct http_client_request **req; + const struct http_client_settings *set = &conn->client->set; + struct http_client_request *req, **req_idx; if (!array_is_created(&conn->request_wait_list)) return; - array_foreach_modifiable(&conn->request_wait_list, req) { - if ((*req)->state < HTTP_REQUEST_STATE_FINISHED) - http_client_request_retry(*req, status, error); - http_client_request_unref(req); + if (set->no_auto_retry) { + http_client_connection_debug(conn, + "Aborting pending requests with error"); + } else { + http_client_connection_debug(conn, + "Retrying pending requests"); + } + + array_foreach_modifiable(&conn->request_wait_list, req_idx) { + req = *req_idx; + /* drop reference from connection */ + req->conn = NULL; + if (!http_client_request_unref(req_idx)) + continue; + /* retry the request, which may drop it */ + if (req->state < HTTP_REQUEST_STATE_FINISHED) { + if (set->no_auto_retry) + http_client_request_error(&req, status, error); + else + http_client_request_retry(req, status, error); + } } array_clear(&conn->request_wait_list); } @@ -85,15 +101,20 @@ static void http_client_connection_server_close(struct http_client_connection **_conn) { struct http_client_connection *conn = *_conn; - struct http_client_request **req; + struct http_client_request *req, **req_idx; http_client_connection_debug(conn, "Server explicitly closed connection"); - array_foreach_modifiable(&conn->request_wait_list, req) { - if ((*req)->state < HTTP_REQUEST_STATE_FINISHED) - http_client_request_resubmit(*req); - http_client_request_unref(req); + array_foreach_modifiable(&conn->request_wait_list, req_idx) { + req = *req_idx; + /* drop reference from connection */ + req->conn = NULL; + if (!http_client_request_unref(req_idx)) + continue; + /* resubmit the request, which may drop it */ + if (req->state < HTTP_REQUEST_STATE_FINISHED) + http_client_request_resubmit(req); } array_clear(&conn->request_wait_list); @@ -108,14 +129,19 @@ http_client_connection_abort_error(struct http_client_connection **_conn, unsigned int status, const char *error) { struct http_client_connection *conn = *_conn; - struct http_client_request **req; + struct http_client_request *req, **req_idx; http_client_connection_debug(conn, "Aborting connection: %s", error); - array_foreach_modifiable(&conn->request_wait_list, req) { - i_assert((*req)->submitted); - http_client_request_error(*req, status, error); - http_client_request_unref(req); + array_foreach_modifiable(&conn->request_wait_list, req_idx) { + req = *req_idx; + i_assert(req->submitted); + /* drop reference from connection */ + req->conn = NULL; + if (!http_client_request_unref(req_idx)) + continue; + /* drop request if not already aborted */ + http_client_request_error(&req, status, error); } array_clear(&conn->request_wait_list); http_client_connection_close(_conn); @@ -124,25 +150,33 @@ http_client_connection_abort_error(struct http_client_connection **_conn, static void http_client_connection_abort_any_requests(struct http_client_connection *conn) { - struct http_client_request **req; + struct http_client_request *req, **req_idx; if (array_is_created(&conn->request_wait_list)) { - array_foreach_modifiable(&conn->request_wait_list, req) { - i_assert((*req)->submitted); - http_client_request_error(*req, + array_foreach_modifiable(&conn->request_wait_list, req_idx) { + req = *req_idx; + i_assert(req->submitted); + /* drop reference from connection */ + req->conn = NULL; + if (!http_client_request_unref(req_idx)) + continue; + /* drop request if not already aborted */ + http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Aborting"); - http_client_request_unref(req); } array_clear(&conn->request_wait_list); } if (conn->pending_request != NULL) { - struct http_client_request *pending_req = conn->pending_request; - conn->pending_request = NULL; - http_client_request_error(pending_req, - HTTP_CLIENT_REQUEST_ERROR_ABORTED, - "Aborting"); - http_client_request_unref(&pending_req); + req = conn->pending_request; + /* drop reference from connection */ + req->conn = NULL; + if (http_client_request_unref(&conn->pending_request)) { + /* drop request if not already aborted */ + http_client_request_error(&req, + HTTP_CLIENT_REQUEST_ERROR_ABORTED, + "Aborting"); + } } } @@ -211,7 +245,7 @@ http_client_connection_abort_temp_error(struct http_client_connection **_conn, http_client_connection_close(_conn); } -bool http_client_connection_is_ready(struct http_client_connection *conn) +int http_client_connection_check_ready(struct http_client_connection *conn) { int ret; @@ -221,14 +255,14 @@ bool http_client_connection_is_ready(struct http_client_connection *conn) this way, but theoretically we could, although that would add quite a bit of complexity. */ - return FALSE; + return 0; } if (!conn->connected || conn->output_locked || conn->output_broken || conn->close_indicated || conn->tunneling || http_client_connection_count_pending(conn) >= conn->client->set.max_pipelined_requests) - return FALSE; + return 0; if (conn->last_ioloop != NULL && conn->last_ioloop != current_ioloop) { conn->last_ioloop = current_ioloop; @@ -246,10 +280,10 @@ bool http_client_connection_is_ready(struct http_client_connection *conn) stream_errno != 0 ? i_stream_get_error(conn->conn.input) : "EOF")); - return FALSE; + return -1; } } - return TRUE; + return 1; } static void @@ -384,10 +418,14 @@ int http_client_connection_next_request(struct http_client_connection *conn) struct http_client_request *req = NULL; const char *error; bool pipelined; + int ret; - if (!http_client_connection_is_ready(conn)) { - http_client_connection_debug(conn, "Not ready for next request"); - return 0; + if ((ret=http_client_connection_check_ready(conn)) <= 0) { + if (ret == 0) { + http_client_connection_debug(conn, + "Not ready for next request"); + } + return ret; } /* claim request, but no urgent request can be second in line */ @@ -402,12 +440,13 @@ int http_client_connection_next_request(struct http_client_connection *conn) if (conn->to_idle != NULL) timeout_remove(&conn->to_idle); - req->conn = conn; req->payload_sync_continue = FALSE; if (conn->peer->no_payload_sync) req->payload_sync = FALSE; + /* add request to wait list and add a reference */ array_append(&conn->request_wait_list, &req, 1); + req->conn = conn; http_client_request_ref(req); http_client_connection_debug(conn, "Claimed request %s", @@ -519,33 +558,37 @@ static void http_client_payload_destroyed(struct http_client_request *req) the payload. make sure here that it's switched back. */ net_set_nonblock(conn->conn.fd_in, TRUE); + /* drop reference from connection */ req->conn = NULL; - conn->incoming_payload = NULL; - conn->pending_request = NULL; - http_client_connection_ref(conn); - http_client_request_finish(req); + if (http_client_request_unref(&conn->pending_request)) { + /* finish request if not already aborted */ + http_client_request_finish(req); + } - /* room for new requests */ - if (http_client_connection_is_ready(conn)) - http_client_peer_trigger_request_handler(conn->peer); + conn->incoming_payload = NULL; /* input stream may have pending input. make sure input handler gets called (but don't do it directly, since we get get here somewhere from the API user's code, which we can't really know what state it is in). this call also triggers sending a new request if necessary. */ - conn->to_input = - timeout_add_short(0, http_client_payload_destroyed_timeout, conn); + if (!conn->disconnected) { + conn->to_input = timeout_add_short + (0, http_client_payload_destroyed_timeout, conn); + } - http_client_request_unref(&req); - http_client_connection_unref(&conn); + /* room for new requests */ + if (http_client_connection_check_ready(conn) > 0) + http_client_peer_trigger_request_handler(conn->peer); } static bool -http_client_connection_return_response(struct http_client_request *req, +http_client_connection_return_response( + struct http_client_connection *conn, + struct http_client_request *req, struct http_response *response) { - struct http_client_connection *conn = req->conn, *tmp_conn; + struct http_client_connection *tmp_conn; struct istream *payload; bool retrying, ret; @@ -577,9 +620,9 @@ http_client_connection_return_response(struct http_client_request *req, http_client_connection_ref(conn); retrying = !http_client_request_callback(req, response); tmp_conn = conn; - if (!http_client_connection_unref(&tmp_conn)) { + if (!http_client_connection_unref(&tmp_conn) || + conn->disconnected) { /* the callback managed to get this connection destroyed */ - req->conn = NULL; if (!retrying) http_client_request_finish(req); http_client_request_unref(&req); @@ -606,6 +649,9 @@ http_client_connection_return_response(struct http_client_request *req, req->state = HTTP_REQUEST_STATE_PAYLOAD_IN; payload = response->payload; response->payload = NULL; + + /* maintain request reference while payload is pending */ + req->conn = conn; conn->pending_request = req; /* request is dereferenced in payload destroy callback */ @@ -616,7 +662,6 @@ http_client_connection_return_response(struct http_client_request *req, http_client_payload_finished(conn); } } else { - req->conn = NULL; http_client_request_finish(req); http_client_request_unref(&req); } @@ -804,6 +849,7 @@ static void http_client_connection_input(struct connection *_conn) /* remove request from queue */ array_delete(&conn->request_wait_list, 0, 1); + req->conn = NULL; aborted = (req->state == HTTP_REQUEST_STATE_ABORTED); req_ref = req; if (!http_client_request_unref(&req_ref)) { @@ -819,11 +865,10 @@ static void http_client_connection_input(struct connection *_conn) /* response cannot be 2xx if request payload was not completely sent */ if (early && response.status / 100 == 2) { - http_client_request_error(req, + http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE, "Server responded with success response " "before all payload was sent"); - http_client_request_unref(&req); http_client_connection_close(&conn); return; } @@ -871,8 +916,8 @@ static void http_client_connection_input(struct connection *_conn) if (!handled) { /* response for application */ - i_assert(req->conn == conn); - if (!http_client_connection_return_response(req, &response)) + if (!http_client_connection_return_response + (conn, req, &response)) return; } } @@ -932,7 +977,7 @@ static void http_client_connection_input(struct connection *_conn) conn->peer->allows_pipelining = TRUE; /* room for new requests */ - if (http_client_connection_is_ready(conn)) + if (http_client_connection_check_ready(conn) > 0) http_client_peer_trigger_request_handler(conn->peer); } } @@ -991,7 +1036,7 @@ int http_client_connection_output(struct http_client_connection *conn) } if (!conn->output_locked) { /* room for new requests */ - if (http_client_connection_is_ready(conn)) + if (http_client_connection_check_ready(conn) > 0) http_client_peer_trigger_request_handler(conn->peer); } } @@ -1057,14 +1102,13 @@ http_client_connection_ready(struct http_client_connection *conn) struct http_response response; http_client_request_ref(req); - req->conn = conn; conn->tunneling = TRUE; memset(&response, 0, sizeof(response)); response.status = 200; response.reason = "OK"; - (void)http_client_connection_return_response(req, &response); + (void)http_client_connection_return_response(conn, req, &response); http_client_request_unref(&req); return; } @@ -1152,6 +1196,7 @@ http_client_connection_connected(struct connection *_conn, bool success) { struct http_client_connection *conn = (struct http_client_connection *)_conn; + const struct http_client_settings *set = &conn->client->set; const char *error; if (!success) { @@ -1160,6 +1205,20 @@ http_client_connection_connected(struct connection *_conn, bool success) } else { conn->connected_timestamp = ioloop_timeval; http_client_connection_debug(conn, "Connected"); + + if (set->socket_send_buffer_size > 0) { + if (net_set_send_buffer_size(_conn->fd_out, + set->socket_send_buffer_size) < 0) + i_error("net_set_send_buffer_size(%"PRIuSIZE_T") failed: %m", + set->socket_send_buffer_size); + } + if (set->socket_recv_buffer_size > 0) { + if (net_set_recv_buffer_size(_conn->fd_in, + set->socket_recv_buffer_size) < 0) + i_error("net_set_recv_buffer_size(%"PRIuSIZE_T") failed: %m", + set->socket_recv_buffer_size); + } + if (http_client_peer_addr_is_https(&conn->peer->addr)) { if (http_client_connection_ssl_init(conn, &error) < 0) { http_client_peer_connection_failure(conn->peer, error); @@ -1369,6 +1428,16 @@ void http_client_connection_ref(struct http_client_connection *conn) static void http_client_connection_disconnect(struct http_client_connection *conn) { + struct http_client_peer *peer = conn->peer; + ARRAY_TYPE(http_client_connection) *conn_arr; + struct http_client_connection *const *conn_idx; + + if (conn->disconnected) + return; + conn->disconnected = TRUE; + + http_client_connection_debug(conn, "Connection disconnect"); + conn->closing = TRUE; conn->connected = FALSE; @@ -1402,14 +1471,23 @@ http_client_connection_disconnect(struct http_client_connection *conn) timeout_remove(&conn->to_idle); if (conn->to_response != NULL) timeout_remove(&conn->to_response); + + /* remove this connection from the list */ + conn_arr = &peer->conns; + array_foreach(conn_arr, conn_idx) { + if (*conn_idx == conn) { + array_delete(conn_arr, array_foreach_idx(conn_arr, conn_idx), 1); + break; + } + } + + if (conn->connect_succeeded) + http_client_peer_connection_lost(peer); } bool http_client_connection_unref(struct http_client_connection **_conn) { struct http_client_connection *conn = *_conn; - struct http_client_connection *const *conn_idx; - ARRAY_TYPE(http_client_connection) *conn_arr; - struct http_client_peer *peer = conn->peer; i_assert(conn->refcount > 0); @@ -1422,6 +1500,13 @@ bool http_client_connection_unref(struct http_client_connection **_conn) http_client_connection_disconnect(conn); + i_assert(conn->io_req_payload == NULL); + i_assert(conn->to_requests == NULL); + i_assert(conn->to_connect == NULL); + i_assert(conn->to_input == NULL); + i_assert(conn->to_idle == NULL); + i_assert(conn->to_response == NULL); + if (array_is_created(&conn->request_wait_list)) array_free(&conn->request_wait_list); @@ -1430,17 +1515,6 @@ bool http_client_connection_unref(struct http_client_connection **_conn) if (conn->connect_initialized) connection_deinit(&conn->conn); - /* remove this connection from the list */ - conn_arr = &conn->peer->conns; - array_foreach(conn_arr, conn_idx) { - if (*conn_idx == conn) { - array_delete(conn_arr, array_foreach_idx(conn_arr, conn_idx), 1); - break; - } - } - - if (conn->connect_succeeded) - http_client_peer_connection_lost(peer); i_free(conn); return FALSE; } @@ -1456,6 +1530,17 @@ void http_client_connection_close(struct http_client_connection **_conn) http_client_connection_unref(_conn); } +void http_client_connection_peer_closed(struct http_client_connection **_conn) +{ + struct http_client_connection *conn = *_conn; + + http_client_connection_debug(conn, "Peer closed"); + http_client_connection_disconnect(conn); + + if (http_client_connection_unref(_conn)) + conn->peer = NULL; +} + void http_client_connection_switch_ioloop(struct http_client_connection *conn) { if (conn->io_req_payload != NULL) diff --git a/src/lib-http/http-client-host.c b/src/lib-http/http-client-host.c index f0a0558e4fa..5c3c7d242c4 100644 --- a/src/lib-http/http-client-host.c +++ b/src/lib-http/http-client-host.c @@ -210,7 +210,7 @@ void http_client_host_submit_request(struct http_client_host *host, if (http_client_peer_addr_is_https(&addr) && host->client->ssl_ctx == NULL) { if (http_client_init_ssl_ctx(host->client, &error) < 0) { - http_client_request_error(req, + http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, error); return; } diff --git a/src/lib-http/http-client-peer.c b/src/lib-http/http-client-peer.c index ed4e305dd35..d963e03f2b0 100644 --- a/src/lib-http/http-client-peer.c +++ b/src/lib-http/http-client-peer.c @@ -128,7 +128,7 @@ http_client_peer_connect_backoff(struct http_client_peer *peer) timeout_remove(&peer->to_backoff); if (array_count(&peer->queues) == 0) { - http_client_peer_free(&peer); + http_client_peer_close(&peer); return; } @@ -183,23 +183,6 @@ bool http_client_peer_is_connected(struct http_client_peer *peer) return FALSE; } -static void -http_client_peer_disconnect(struct http_client_peer *peer) -{ - struct http_client_connection **conn; - ARRAY_TYPE(http_client_connection) conns; - - http_client_peer_debug(peer, "Peer disconnect"); - - /* make a copy of the connection array; freed connections modify it */ - t_array_init(&conns, array_count(&peer->conns)); - array_copy(&conns.arr, 0, &peer->conns.arr, 0, array_count(&peer->conns)); - array_foreach_modifiable(&conns, conn) { - http_client_connection_unref(conn); - } - i_assert(array_count(&peer->conns) == 0); -} - static unsigned int http_client_peer_requests_pending(struct http_client_peer *peer, unsigned int *num_urgent_r) @@ -225,7 +208,7 @@ static void http_client_peer_check_idle(struct http_client_peer *peer) if (array_count(&peer->conns) == 0 && http_client_peer_requests_pending(peer, &num_urgent) == 0) { /* no connections or pending requests; die immediately */ - http_client_peer_free(&peer); + http_client_peer_close(&peer); return; } @@ -247,6 +230,7 @@ http_client_peer_handle_requests_real(struct http_client_peer *peer) struct _conn_available *conn_avail_idx; unsigned int connecting, closing, idle; unsigned int num_pending, num_urgent, new_connections, working_conn_count; + struct http_client_peer *tmp_peer; bool statistics_dirty = TRUE; /* FIXME: limit the number of requests handled in one run to prevent @@ -258,7 +242,7 @@ http_client_peer_handle_requests_real(struct http_client_peer *peer) http_client_peer_debug(peer, "Peer no longer used; will now disconnect " "(%u connections exist)", array_count(&peer->conns)); - http_client_peer_disconnect(peer); + http_client_peer_close(&peer); return; } @@ -272,6 +256,7 @@ http_client_peer_handle_requests_real(struct http_client_peer *peer) return; } + http_client_peer_ref(peer); peer->handling_requests = TRUE; t_array_init(&conns_avail, array_count(&peer->conns)); do { @@ -283,9 +268,12 @@ http_client_peer_handle_requests_real(struct http_client_peer *peer) /* gather connection statistics */ array_foreach(&peer->conns, conn_idx) { struct http_client_connection *conn = *conn_idx; + int ret; - http_client_connection_ref(conn); - if (http_client_connection_is_ready(conn)) { + if ((ret=http_client_connection_check_ready(conn)) < 0) { + conn_lost = TRUE; + break; + } else if (ret > 0) { struct _conn_available *conn_avail; unsigned int insert_idx, pending_requests; @@ -308,11 +296,6 @@ http_client_peer_handle_requests_real(struct http_client_peer *peer) if (pending_requests == 0) idle++; } - if (!http_client_connection_unref(&conn)) { - conn_lost = TRUE; - break; - } - conn = *conn_idx; /* count the number of connecting and closing connections */ if (conn->closing) closing++; @@ -364,6 +347,10 @@ http_client_peer_handle_requests_real(struct http_client_peer *peer) break; } } while (statistics_dirty); + + tmp_peer = peer; + if (!http_client_peer_unref(&tmp_peer)) + return; peer->handling_requests = FALSE; if (num_pending == 0) @@ -494,6 +481,7 @@ http_client_peer_create(struct http_client *client, struct http_client_peer *peer; peer = i_new(struct http_client_peer, 1); + peer->refcount = 1; peer->client = client; peer->addr = *addr; @@ -528,33 +516,72 @@ http_client_peer_create(struct http_client *client, return peer; } -void http_client_peer_free(struct http_client_peer **_peer) +static void +http_client_peer_disconnect(struct http_client_peer *peer) { - struct http_client_peer *peer = *_peer; - - *_peer = NULL; + struct http_client_connection **conn; + ARRAY_TYPE(http_client_connection) conns; - if (peer->destroyed) + if (peer->disconnected) return; - peer->destroyed = TRUE; + peer->disconnected = TRUE; - http_client_peer_debug(peer, "Peer destroy"); + http_client_peer_debug(peer, "Peer disconnect"); + + /* make a copy of the connection array; freed connections modify it */ + t_array_init(&conns, array_count(&peer->conns)); + array_copy(&conns.arr, 0, &peer->conns.arr, 0, array_count(&peer->conns)); + array_foreach_modifiable(&conns, conn) { + http_client_connection_peer_closed(conn); + } + i_assert(array_count(&peer->conns) == 0); if (peer->to_req_handling != NULL) timeout_remove(&peer->to_req_handling); if (peer->to_backoff != NULL) timeout_remove(&peer->to_backoff); - http_client_peer_disconnect(peer); - array_free(&peer->conns); - array_free(&peer->queues); - hash_table_remove (peer->client->peers, (const struct http_client_peer_addr *)&peer->addr); DLLIST_REMOVE(&peer->client->peers_list, peer); +} + +void http_client_peer_ref(struct http_client_peer *peer) +{ + peer->refcount++; +} + +bool http_client_peer_unref(struct http_client_peer **_peer) +{ + struct http_client_peer *peer = *_peer; + i_assert(peer->refcount > 0); + + *_peer = NULL; + + if (--peer->refcount > 0) + return TRUE; + + http_client_peer_debug(peer, "Peer destroy"); + + http_client_peer_disconnect(peer); + + array_free(&peer->conns); + array_free(&peer->queues); i_free(peer->addr_name); i_free(peer); + return FALSE; +} + +void http_client_peer_close(struct http_client_peer **_peer) +{ + struct http_client_peer *peer = *_peer; + + http_client_peer_debug(peer, "Peer close"); + + http_client_peer_disconnect(peer); + + (void)http_client_peer_unref(_peer); } struct http_client_peer * @@ -604,7 +631,7 @@ void http_client_peer_unlink_queue(struct http_client_peer *peer, http_client_peer_trigger_request_handler(peer); } else { /* drop peer immediately */ - http_client_peer_free(&peer); + http_client_peer_close(&peer); } } return; @@ -696,32 +723,29 @@ void http_client_peer_connection_failure(struct http_client_peer *peer, void http_client_peer_connection_lost(struct http_client_peer *peer) { - unsigned int num_urgent; + unsigned int num_pending, num_urgent; /* we get here when an already connected connection fails. if the connect itself fails, http_client_peer_connection_failure() is called instead. */ - if (peer->destroyed) + if (peer->disconnected) return; - http_client_peer_debug(peer, "Lost a connection (%d connections left)", - array_count(&peer->conns)); + num_pending = http_client_peer_requests_pending(peer, &num_urgent); + + http_client_peer_debug(peer, + "Lost a connection " + "(%d connections left, %u requests pending, %u requests urgent)", + array_count(&peer->conns), num_pending, num_urgent); if (peer->handling_requests) { /* we got here from the request handler loop */ return; } - /* check if peer is still relevant */ - if (array_count(&peer->conns) == 0 && - http_client_peer_requests_pending(peer, &num_urgent) == 0) { - http_client_peer_free(&peer); - return; - } - /* if there are pending requests for this peer, create a new connection - for them. */ + for them. if not, this peer will wind itself down. */ http_client_peer_trigger_request_handler(peer); } diff --git a/src/lib-http/http-client-private.h b/src/lib-http/http-client-private.h index f34a4ceb1d8..f62630ff432 100644 --- a/src/lib-http/http-client-private.h +++ b/src/lib-http/http-client-private.h @@ -119,9 +119,11 @@ struct http_client_request { unsigned int payload_wait:1; unsigned int urgent:1; unsigned int submitted:1; + unsigned int listed:1; unsigned int connect_tunnel:1; unsigned int connect_direct:1; unsigned int ssl_tunnel:1; + unsigned int preserve_exact_reason:1; }; struct http_client_connection { @@ -157,6 +159,7 @@ struct http_client_connection { unsigned int connect_initialized:1; /* connection was initialized */ unsigned int connect_succeeded:1; unsigned int closing:1; + unsigned int disconnected:1; unsigned int close_indicated:1; unsigned int output_locked:1; /* output is locked; no pipelining */ unsigned int output_broken:1; /* output is broken; no more requests */ @@ -164,6 +167,7 @@ struct http_client_connection { }; struct http_client_peer { + unsigned int refcount; struct http_client_peer_addr addr; char *addr_name; @@ -184,7 +188,7 @@ struct http_client_peer { struct timeout *to_backoff; unsigned int backoff_time_msecs; - unsigned int destroyed:1; /* peer is being destroyed */ + unsigned int disconnected:1; /* peer is already disconnected */ unsigned int no_payload_sync:1; /* expect: 100-continue failed before */ unsigned int seen_100_response:1;/* expect: 100-continue succeeded before */ unsigned int allows_pipelining:1;/* peer is known to allow persistent @@ -207,6 +211,7 @@ struct http_client_queue { connected IP */ unsigned int ips_connect_start_idx; + struct timeval first_connect_time; unsigned int connect_attempts; /* peers we are trying to connect to; @@ -294,7 +299,7 @@ void http_client_request_resubmit(struct http_client_request *req); void http_client_request_retry(struct http_client_request *req, unsigned int status, const char *error); void http_client_request_error_delayed(struct http_client_request **_req); -void http_client_request_error(struct http_client_request *req, +void http_client_request_error(struct http_client_request **req, unsigned int status, const char *error); void http_client_request_redirect(struct http_client_request *req, unsigned int status, const char *location); @@ -308,6 +313,9 @@ void http_client_connection_ref(struct http_client_connection *conn); /* Returns FALSE if unrefing destroyed the connection entirely */ bool http_client_connection_unref(struct http_client_connection **_conn); void http_client_connection_close(struct http_client_connection **_conn); + +void http_client_connection_peer_closed(struct http_client_connection **_conn); + int http_client_connection_output(struct http_client_connection *conn); void http_client_connection_start_request_timeout( struct http_client_connection *conn); @@ -317,7 +325,7 @@ void http_client_connection_stop_request_timeout( struct http_client_connection *conn); unsigned int http_client_connection_count_pending(struct http_client_connection *conn); -bool http_client_connection_is_ready(struct http_client_connection *conn); +int http_client_connection_check_ready(struct http_client_connection *conn); bool http_client_connection_is_idle(struct http_client_connection *conn); int http_client_connection_next_request(struct http_client_connection *conn); void http_client_connection_check_idle(struct http_client_connection *conn); @@ -335,7 +343,10 @@ int http_client_peer_addr_cmp struct http_client_peer * http_client_peer_get(struct http_client *client, const struct http_client_peer_addr *addr); -void http_client_peer_free(struct http_client_peer **_peer); +void http_client_peer_ref(struct http_client_peer *peer); +bool http_client_peer_unref(struct http_client_peer **_peer); +void http_client_peer_close(struct http_client_peer **_peer); + bool http_client_peer_have_queue(struct http_client_peer *peer, struct http_client_queue *queue); void http_client_peer_link_queue(struct http_client_peer *peer, diff --git a/src/lib-http/http-client-queue.c b/src/lib-http/http-client-queue.c index 7b2bbff90b2..7b7e921b133 100644 --- a/src/lib-http/http-client-queue.c +++ b/src/lib-http/http-client-queue.c @@ -154,7 +154,7 @@ void http_client_queue_fail(struct http_client_queue *queue, t_array_init(&treqs, array_count(req_arr)); array_copy(&treqs.arr, 0, &req_arr->arr, 0, array_count(req_arr)); array_foreach_modifiable(&treqs, req_idx) { - http_client_request_error(*req_idx, status, error); + http_client_request_error(req_idx, status, error); } /* all queues should be empty now... unless new requests were submitted @@ -282,7 +282,8 @@ void http_client_queue_connection_setup(struct http_client_queue *queue) http_client_peer_addr2str(addr), ssl); array_append(&queue->pending_peers, &peer, 1); - queue->connect_attempts++; + if (queue->connect_attempts++ == 0) + queue->first_connect_time = ioloop_timeval; } /* start soft connect time-out (but only if we have another IP left) */ @@ -401,6 +402,13 @@ http_client_queue_connection_failure(struct http_client_queue *queue, queue->connect_attempts >= set->max_connect_attempts) { http_client_queue_debug(queue, "Failed to set up any connection; failing all queued requests"); + if (queue->connect_attempts > 1) { + unsigned int total_msecs = + timeval_diff_msecs(&ioloop_timeval, &queue->first_connect_time); + reason = t_strdup_printf("%s (%u attempts in %u.%03u secs)", + reason, queue->connect_attempts, + total_msecs/1000, total_msecs%1000); + } queue->connect_attempts = 0; http_client_queue_fail(queue, HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, reason); @@ -528,7 +536,7 @@ http_client_queue_request_timeout(struct http_client_queue *queue) http_client_queue_debug(queue, "Request %s timed out", http_client_request_label(req)); - http_client_request_error(req, + http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT, "Timed out"); } @@ -631,6 +639,7 @@ http_client_queue_delay_timeout(struct http_client_queue *queue) struct http_client_request *const *reqs; unsigned int count, i, finished; + timeout_remove(&queue->to_delayed); io_loop_time_refresh(); finished = 0; @@ -734,6 +743,12 @@ void http_client_queue_submit_request(struct http_client_queue *queue, if (timeval_cmp_margin(&req->release_time, &ioloop_timeval, TIMEOUT_CMP_MARGIN_USECS) > 0) { + http_client_queue_debug(queue, + "Delayed request %s%s submitted (time remaining: %d msecs)", + http_client_request_label(req), + (req->urgent ? " (urgent)" : ""), + timeval_diff_msecs(&req->release_time, &ioloop_timeval)); + (void)array_bsearch_insert_pos(&queue->delayed_requests, &req, http_client_queue_delayed_cmp, &insert_idx); array_insert(&queue->delayed_requests, insert_idx, &req, 1); diff --git a/src/lib-http/http-client-request.c b/src/lib-http/http-client-request.c index ce1a00d45ac..f1041d7b1a0 100644 --- a/src/lib-http/http-client-request.c +++ b/src/lib-http/http-client-request.c @@ -55,7 +55,7 @@ http_client_request_debug(struct http_client_request *req, * Request */ -static void +static bool http_client_request_send_error(struct http_client_request *req, unsigned int status, const char *error); @@ -150,6 +150,32 @@ http_client_request_connect_ip(struct http_client *client, return req; } +static void +http_client_request_add(struct http_client_request *req) +{ + struct http_client *client = req->client; + + DLLIST_PREPEND(&client->requests_list, req); + client->requests_count++; + req->listed = TRUE; +} + +static void +http_client_request_remove(struct http_client_request *req) +{ + struct http_client *client = req->client; + + if (req->listed) { + /* only decrease pending request counter if this request was submitted */ + DLLIST_REMOVE(&client->requests_list, req); + client->requests_count--; + } + req->listed = FALSE; + + if (client->requests_count == 0 && client->ioloop != NULL) + io_loop_stop(client->ioloop); +} + void http_client_request_ref(struct http_client_request *req) { i_assert(req->refcount > 0); @@ -172,7 +198,7 @@ bool http_client_request_unref(struct http_client_request **_req) client->requests_count); /* cannot be destroyed while it is still pending */ - i_assert(req->conn == NULL || req->conn->pending_request == NULL); + i_assert(req->conn == NULL); if (req->queue != NULL) http_client_queue_drop_request(req->queue, req); @@ -182,11 +208,7 @@ bool http_client_request_unref(struct http_client_request **_req) req->destroy_callback = NULL; } - /* only decrease pending request counter if this request was submitted */ - if (req->submitted) { - DLLIST_REMOVE(&client->requests_list, req); - client->requests_count--; - } + http_client_request_remove(req); if (client->requests_count == 0 && client->ioloop != NULL) io_loop_stop(client->ioloop); @@ -213,6 +235,10 @@ void http_client_request_destroy(struct http_client_request **_req) http_client_request_debug(req, "Destroy (requests left=%d)", client->requests_count); + if (req->state < HTTP_REQUEST_STATE_FINISHED) + req->state = HTTP_REQUEST_STATE_ABORTED; + req->callback = NULL; + if (req->queue != NULL) http_client_queue_drop_request(req->queue, req); @@ -222,6 +248,7 @@ void http_client_request_destroy(struct http_client_request **_req) req->destroy_callback = NULL; callback(req->destroy_context); } + http_client_request_remove(req); http_client_request_unref(&req); } @@ -246,6 +273,11 @@ void http_client_request_set_urgent(struct http_client_request *req) req->urgent = TRUE; } +void http_client_request_set_preserve_exact_reason(struct http_client_request *req) +{ + req->preserve_exact_reason = TRUE; +} + void http_client_request_add_header(struct http_client_request *req, const char *key, const char *value) { @@ -356,6 +388,23 @@ void http_client_request_set_payload(struct http_client_request *req, req->payload_sync = TRUE; } +void http_client_request_set_payload_data(struct http_client_request *req, + const unsigned char *data, size_t size) +{ + struct istream *input; + unsigned char *payload_data; + + if (size == 0) + return; + + payload_data = p_malloc(req->pool, size); + memcpy(payload_data, data, size); + input = i_stream_create_from_data(payload_data, size); + + http_client_request_set_payload(req, input, FALSE); + i_stream_unref(&input); +} + void http_client_request_set_timeout_msecs(struct http_client_request *req, unsigned int msecs) { @@ -540,16 +589,13 @@ static void http_client_request_do_submit(struct http_client_request *req) void http_client_request_submit(struct http_client_request *req) { - struct http_client *client = req->client; - req->submit_time = ioloop_timeval; http_client_request_do_submit(req); http_client_request_debug(req, "Submitted"); req->submitted = TRUE; - DLLIST_PREPEND(&client->requests_list, req); - client->requests_count++; + http_client_request_add(req); } void @@ -766,18 +812,18 @@ int http_client_request_send_more(struct http_client_request *req, o_stream_set_max_buffer_size(output, (size_t)-1); if (req->payload_input->stream_errno != 0) { - /* the payload stream assigned to this request is broken, - fail this the request immediately */ - http_client_request_send_error(req, - HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD, - "Broken payload stream"); - /* we're in the middle of sending a request, so the connection will also have to be aborted */ errno = req->payload_input->stream_errno; *error_r = t_strdup_printf("read(%s) failed: %s", i_stream_get_name(req->payload_input), i_stream_get_error(req->payload_input)); + + /* the payload stream assigned to this request is broken, + fail this the request immediately */ + http_client_request_error(&req, + HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD, + "Broken payload stream"); return -1; } else if (output->stream_errno != 0) { /* failed to send request */ @@ -813,7 +859,7 @@ int http_client_request_send_more(struct http_client_request *req, /* finished sending payload */ http_client_request_finish_payload_out(req); } - } else if (i_stream_get_data_size(req->payload_input) > 0) { + } else if (i_stream_have_bytes_left(req->payload_input)) { /* output is blocking (server needs to act; enable timeout) */ conn->output_locked = TRUE; if (!pipelined) @@ -999,7 +1045,18 @@ bool http_client_request_callback(struct http_client_request *req, req->callback = NULL; if (callback != NULL) { - callback(response, req->context); + struct http_response response_copy = *response; + + if (req->attempts > 0 && !req->preserve_exact_reason) { + unsigned int total_msecs = + timeval_diff_msecs(&ioloop_timeval, &req->submit_time); + response_copy.reason = t_strdup_printf( + "%s (%u attempts in %u.%03u secs)", + response_copy.reason, req->attempts, + total_msecs/1000, total_msecs%1000); + } + + callback(&response_copy, req->context); if (req->attempts != orig_attempts) { /* retrying */ req->callback = callback; @@ -1014,12 +1071,13 @@ bool http_client_request_callback(struct http_client_request *req, return TRUE; } -static void +static bool http_client_request_send_error(struct http_client_request *req, unsigned int status, const char *error) { http_client_request_callback_t *callback; bool sending = (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT); + unsigned int orig_attempts = req->attempts; req->state = HTTP_REQUEST_STATE_ABORTED; @@ -1031,35 +1089,46 @@ http_client_request_send_error(struct http_client_request *req, http_response_init(&response, status, error); (void)callback(&response, req->context); - /* release payload early (prevents server/client deadlock in proxy) */ - if (!sending && req->payload_input != NULL) - i_stream_unref(&req->payload_input); + if (req->attempts != orig_attempts) { + /* retrying */ + req->callback = callback; + http_client_request_resubmit(req); + return FALSE; + } else { + /* release payload early (prevents server/client deadlock in proxy) */ + if (!sending && req->payload_input != NULL) + i_stream_unref(&req->payload_input); + } } if (req->payload_wait && req->client->ioloop != NULL) io_loop_stop(req->client->ioloop); + return TRUE; } void http_client_request_error_delayed(struct http_client_request **_req) { struct http_client_request *req = *_req; + bool destroy; i_assert(req->state == HTTP_REQUEST_STATE_ABORTED); *_req = NULL; i_assert(req->delayed_error != NULL && req->delayed_error_status != 0); - http_client_request_send_error(req, req->delayed_error_status, + destroy = http_client_request_send_error(req, req->delayed_error_status, req->delayed_error); if (req->queue != NULL) http_client_queue_drop_request(req->queue, req); - http_client_request_destroy(&req); + if (destroy) + http_client_request_destroy(&req); } -void http_client_request_error(struct http_client_request *req, +void http_client_request_error(struct http_client_request **_req, unsigned int status, const char *error) { - if (req->state >= HTTP_REQUEST_STATE_FINISHED) - return; + struct http_client_request *req = *_req; + + i_assert(req->state < HTTP_REQUEST_STATE_FINISHED); req->state = HTTP_REQUEST_STATE_ABORTED; if (req->queue != NULL) @@ -1075,9 +1144,10 @@ void http_client_request_error(struct http_client_request *req, req->delayed_error_status = status; http_client_delay_request_error(req->client, req); } else { - http_client_request_send_error(req, status, error); - http_client_request_destroy(&req); + if (http_client_request_send_error(req, status, error)) + http_client_request_destroy(&req); } + *_req = NULL; } void http_client_request_abort(struct http_client_request **_req) @@ -1109,7 +1179,8 @@ void http_client_request_finish(struct http_client_request *req) if (req->state >= HTTP_REQUEST_STATE_FINISHED) return; - i_assert(req->refcount > 1); + i_assert(req->refcount > 0); + http_client_request_debug(req, "Finished"); req->callback = NULL; @@ -1133,19 +1204,19 @@ void http_client_request_redirect(struct http_client_request *req, /* parse URL */ if (http_url_parse(location, NULL, 0, pool_datastack_create(), &url, &error) < 0) { - http_client_request_error(req, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT, + http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT, t_strdup_printf("Invalid redirect location: %s", error)); return; } if (++req->redirects > req->client->set.max_redirects) { if (req->client->set.max_redirects > 0) { - http_client_request_error(req, + http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT, t_strdup_printf("Redirected more than %d times", req->client->set.max_redirects)); } else { - http_client_request_error(req, + http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT, "Redirect refused"); } @@ -1156,7 +1227,7 @@ void http_client_request_redirect(struct http_client_request *req, if (req->payload_input != NULL && req->payload_size > 0 && status != 303) { if (req->payload_input->v_offset != req->payload_offset && !req->payload_input->seekable) { - http_client_request_error(req, + http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Redirect failed: Cannot resend payload; stream is not seekable"); return; @@ -1219,20 +1290,7 @@ void http_client_request_resubmit(struct http_client_request *req) if (req->payload_input != NULL && req->payload_size > 0) { if (req->payload_input->v_offset != req->payload_offset && !req->payload_input->seekable) { - http_client_request_error(req, - HTTP_CLIENT_REQUEST_ERROR_ABORTED, - "Resubmission failed: Cannot resend payload; stream is not seekable"); - return; - } else { - i_stream_seek(req->payload_input, req->payload_offset); - } - } - - /* rewind payload stream */ - if (req->payload_input != NULL && req->payload_size > 0) { - if (req->payload_input->v_offset != req->payload_offset && - !req->payload_input->seekable) { - http_client_request_error(req, + http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_ABORTED, "Resubmission failed: Cannot resend payload; stream is not seekable"); return; @@ -1255,7 +1313,7 @@ void http_client_request_retry(struct http_client_request *req, unsigned int status, const char *error) { if (!http_client_request_try_retry(req)) - http_client_request_error(req, status, error); + http_client_request_error(&req, status, error); } bool http_client_request_try_retry(struct http_client_request *req) diff --git a/src/lib-http/http-client.c b/src/lib-http/http-client.c index 4ed33452634..14814f8047e 100644 --- a/src/lib-http/http-client.c +++ b/src/lib-http/http-client.c @@ -137,6 +137,7 @@ struct http_client *http_client_init(const struct http_client_settings *set) HTTP_CLIENT_DEFAULT_BACKOFF_MAX_TIME_MSECS : set->connect_backoff_max_time_msecs; client->set.no_auto_redirect = set->no_auto_redirect; + client->set.no_auto_retry = set->no_auto_retry; client->set.no_ssl_tunnel = set->no_ssl_tunnel; client->set.max_redirects = set->max_redirects; client->set.response_hdr_limits = set->response_hdr_limits; @@ -149,6 +150,8 @@ struct http_client *http_client_init(const struct http_client_settings *set) client->set.connect_timeout_msecs = set->connect_timeout_msecs; client->set.soft_connect_timeout_msecs = set->soft_connect_timeout_msecs; client->set.max_auto_retry_delay = set->max_auto_retry_delay; + client->set.socket_send_buffer_size = set->socket_send_buffer_size; + client->set.socket_recv_buffer_size = set->socket_recv_buffer_size; client->set.debug = set->debug; i_array_init(&client->delayed_failing_requests, 1); @@ -165,29 +168,25 @@ struct http_client *http_client_init(const struct http_client_settings *set) void http_client_deinit(struct http_client **_client) { struct http_client *client = *_client; - struct http_client_request *req, *const *req_idx; + struct http_client_request *req; struct http_client_host *host; struct http_client_peer *peer; *_client = NULL; - /* drop delayed failing requests */ - while (array_count(&client->delayed_failing_requests) > 0) { - req_idx = array_idx(&client->delayed_failing_requests, 0); - req = *req_idx; - - i_assert(req->refcount == 1); - http_client_request_error_delayed(&req); + /* destroy requests without calling callbacks */ + req = client->requests_list; + while (req != NULL) { + struct http_client_request *next_req = req->next; + http_client_request_destroy(&req); + req = next_req; } - array_free(&client->delayed_failing_requests); - - if (client->to_failing_requests != NULL) - timeout_remove(&client->to_failing_requests); + i_assert(client->requests_count == 0); /* free peers */ while (client->peers_list != NULL) { peer = client->peers_list; - http_client_peer_free(&peer); + http_client_peer_close(&peer); } hash_table_destroy(&client->peers); @@ -198,6 +197,10 @@ void http_client_deinit(struct http_client **_client) } hash_table_destroy(&client->hosts); + array_free(&client->delayed_failing_requests); + if (client->to_failing_requests != NULL) + timeout_remove(&client->to_failing_requests); + connection_list_deinit(&client->conn_list); if (client->ssl_ctx != NULL) diff --git a/src/lib-http/http-client.h b/src/lib-http/http-client.h index 9f59c58d04d..37bdc0220ed 100644 --- a/src/lib-http/http-client.h +++ b/src/lib-http/http-client.h @@ -72,6 +72,9 @@ struct http_client_settings { /* don't automatically act upon redirect responses */ bool no_auto_redirect; + /* never automatically retry requests */ + bool no_auto_retry; + /* if we use a proxy, delegate SSL negotiation to proxy, rather than creating a CONNECT tunnel through the proxy for the SSL link */ bool no_ssl_tunnel; @@ -126,6 +129,13 @@ struct http_client_settings { is not automatically retried and the response is returned */ unsigned int max_auto_retry_delay; + /* the kernel send/receive buffer sizes used for the connection sockets. + Configuring this is mainly useful for the test suite. The kernel + defaults are used when these settings are 0. */ + size_t socket_send_buffer_size; + size_t socket_recv_buffer_size; + + /* enable logging debug messages */ bool debug; }; @@ -194,6 +204,7 @@ void http_client_request_set_port(struct http_client_request *req, void http_client_request_set_ssl(struct http_client_request *req, bool ssl); void http_client_request_set_urgent(struct http_client_request *req); +void http_client_request_set_preserve_exact_reason(struct http_client_request *req); void http_client_request_add_header(struct http_client_request *req, const char *key, const char *value); @@ -204,6 +215,8 @@ void http_client_request_set_date(struct http_client_request *req, void http_client_request_set_payload(struct http_client_request *req, struct istream *input, bool sync); +void http_client_request_set_payload_data(struct http_client_request *req, + const unsigned char *data, size_t size); void http_client_request_set_timeout_msecs(struct http_client_request *req, unsigned int msecs); diff --git a/src/lib-http/http-response-parser.c b/src/lib-http/http-response-parser.c index 807d75b8fb0..52aeda61f8d 100644 --- a/src/lib-http/http-response-parser.c +++ b/src/lib-http/http-response-parser.c @@ -73,6 +73,9 @@ static int http_response_parse_status(struct http_response_parser *parser) return -1; parser->response_status = (p[0] - '0')*100 + (p[1] - '0')*10 + (p[2] - '0'); + if (parser->response_status < 100 || + parser->response_status >= 600) + return -1; parser->parser.cur += 3; return 1; } diff --git a/src/lib-http/http-server-connection.c b/src/lib-http/http-server-connection.c index 94713e6805b..0b7b8c84afe 100644 --- a/src/lib-http/http-server-connection.c +++ b/src/lib-http/http-server-connection.c @@ -6,6 +6,7 @@ #include "str.h" #include "ioloop.h" #include "istream.h" +#include "istream-timeout.h" #include "ostream.h" #include "connection.h" #include "iostream-rawlog.h" @@ -234,6 +235,7 @@ static void http_server_payload_destroyed(struct http_server_request *req) case HTTP_SERVER_REQUEST_STATE_PAYLOAD_IN: /* finished reading request */ req->state = HTTP_SERVER_REQUEST_STATE_PROCESSING; + http_server_connection_timeout_stop(conn); if (req->response != NULL && req->response->submitted) http_server_request_submit_response(req); break; @@ -301,6 +303,7 @@ static bool http_server_connection_handle_request(struct http_server_connection *conn, struct http_server_request *req) { + const struct http_server_settings *set = &conn->server->set; struct istream *payload; i_assert(!conn->in_req_callback); @@ -317,7 +320,11 @@ http_server_connection_handle_request(struct http_server_connection *conn, /* wrap the stream to capture the destroy event without destroying the actual payload stream. */ conn->incoming_payload = req->req.payload = - i_stream_create_limit(req->req.payload, (uoff_t)-1); + i_stream_create_timeout(req->req.payload, + set->max_client_idle_time_msecs); + /* we've received the request itself, and we can't reset the + timeout during the payload reading. */ + http_server_connection_timeout_stop(conn); } else { conn->incoming_payload = req->req.payload = i_stream_create_from_data("", 0); @@ -811,15 +818,20 @@ http_server_connection_next_response(struct http_server_connection *conn) req = conn->request_queue_head; if (req == NULL) { /* no requests pending */ + http_server_connection_debug(conn, "No more requests pending"); http_server_connection_timeout_start(conn); return FALSE; } if (req->state < HTTP_SERVER_REQUEST_STATE_READY_TO_RESPOND) { if (req->state == HTTP_SERVER_REQUEST_STATE_PROCESSING) { /* server is causing idle time */ + http_server_connection_debug(conn, + "Not ready to respond: Server is processing"); http_server_connection_timeout_stop(conn); } else { /* client is causing idle time */ + http_server_connection_debug(conn, + "Not ready to respond: Waiting for client"); http_server_connection_timeout_start(conn); } @@ -852,6 +864,8 @@ http_server_connection_next_response(struct http_server_connection *conn) i_assert(req->state == HTTP_SERVER_REQUEST_STATE_READY_TO_RESPOND && req->response != NULL); + http_server_connection_debug(conn, + "Sending response"); http_server_connection_timeout_start(conn); http_server_request_ref(req); @@ -940,9 +954,15 @@ int http_server_connection_output(struct http_server_connection *conn) return -1; } else if (conn->io_resp_payload != NULL) { /* server is causing idle time */ + http_server_connection_debug(conn, + "Not ready to continue response: " + "Server is producing response"); http_server_connection_timeout_stop(conn); } else { /* client is causing idle time */ + http_server_connection_debug(conn, + "Not ready to continue response: " + "Waiting for client"); http_server_connection_timeout_start(conn); } } @@ -991,6 +1011,7 @@ http_server_connection_create(struct http_server *server, int fd_in, int fd_out, bool ssl, const struct http_server_callbacks *callbacks, void *context) { + const struct http_server_settings *set = &server->set; struct http_server_connection *conn; static unsigned int id = 0; struct ip_addr addr; @@ -1005,6 +1026,23 @@ http_server_connection_create(struct http_server *server, conn->callbacks = callbacks; conn->context = context; + net_set_nonblock(fd_in, TRUE); + if (fd_in != fd_out) + net_set_nonblock(fd_out, TRUE); + + if (set->socket_send_buffer_size > 0) { + if (net_set_send_buffer_size(fd_out, + set->socket_send_buffer_size) < 0) + i_error("net_set_send_buffer_size(%"PRIuSIZE_T") failed: %m", + set->socket_send_buffer_size); + } + if (set->socket_recv_buffer_size > 0) { + if (net_set_recv_buffer_size(fd_in, + set->socket_recv_buffer_size) < 0) + i_error("net_set_recv_buffer_size(%"PRIuSIZE_T") failed: %m", + set->socket_recv_buffer_size); + } + /* get a name for this connection */ if (fd_in != fd_out || net_getpeername(fd_in, &addr, &port) < 0) { name = t_strdup_printf("[%u]", id); @@ -1062,6 +1100,13 @@ http_server_connection_disconnect(struct http_server_connection *conn, /* preserve statistics */ http_server_connection_update_stats(conn); + if (conn->incoming_payload != NULL) { + /* the stream is still accessed by lib-http caller. */ + i_stream_remove_destroy_callback(conn->incoming_payload, + http_server_payload_destroyed); + conn->incoming_payload = NULL; + } + /* drop all requests before connection is closed */ req = conn->request_queue_head; while (req != NULL) { @@ -1081,13 +1126,6 @@ http_server_connection_disconnect(struct http_server_connection *conn, o_stream_uncork(conn->conn.output); } - if (conn->incoming_payload != NULL) { - /* the stream is still accessed by lib-http caller. */ - i_stream_remove_destroy_callback(conn->incoming_payload, - http_server_payload_destroyed); - conn->incoming_payload = NULL; - } - if (conn->http_parser != NULL) http_request_parser_deinit(&conn->http_parser); connection_disconnect(&conn->conn); diff --git a/src/lib-http/http-server-request.c b/src/lib-http/http-server-request.c index 3d0671ef236..34d7a3744ce 100644 --- a/src/lib-http/http-server-request.c +++ b/src/lib-http/http-server-request.c @@ -128,6 +128,9 @@ void http_server_request_abort(struct http_server_request **_req, struct http_server_request *req = *_req; struct http_server_connection *conn = req->conn; + if (req->state >= HTTP_SERVER_REQUEST_STATE_FINISHED) + return; + http_server_request_debug(req, "Abort"); req->conn = NULL; diff --git a/src/lib-http/http-server-response.c b/src/lib-http/http-server-response.c index f8379364740..aa976640a6d 100644 --- a/src/lib-http/http-server-response.c +++ b/src/lib-http/http-server-response.c @@ -142,8 +142,8 @@ void http_server_response_set_payload(struct http_server_response *resp, resp->payload_input = input; if ((ret = i_stream_get_size(input, TRUE, &resp->payload_size)) <= 0) { if (ret < 0) { - i_error("i_stream_get_size(%s) failed: %m", - i_stream_get_name(input)); + i_error("i_stream_get_size(%s) failed: %s", + i_stream_get_name(input), i_stream_get_error(input)); } resp->payload_size = 0; resp->payload_chunked = TRUE; @@ -500,16 +500,16 @@ int http_server_response_send_more(struct http_server_response *resp, if (resp->payload_input->stream_errno != 0) { /* we're in the middle of sending a response, so the connection will also have to be aborted */ - errno = resp->payload_input->stream_errno; - *error_r = t_strdup_printf("read(%s) failed: %m", - i_stream_get_name(resp->payload_input)); + *error_r = t_strdup_printf("read(%s) failed: %s", + i_stream_get_name(resp->payload_input), + i_stream_get_error(resp->payload_input)); ret = -1; } else if (output->stream_errno != 0) { /* failed to send response */ - errno = output->stream_errno; - if (errno != EPIPE && errno != ECONNRESET) { - *error_r = t_strdup_printf("write(%s) failed: %m", - o_stream_get_name(output)); + if (output->stream_errno != EPIPE && + output->stream_errno != ECONNRESET) { + *error_r = t_strdup_printf("write(%s) failed: %s", + o_stream_get_name(output), o_stream_get_error(output)); } ret = -1; } else { @@ -528,7 +528,7 @@ int http_server_response_send_more(struct http_server_response *resp, } /* finished sending payload */ http_server_response_finish_payload_out(resp); - } else if (i_stream_get_data_size(resp->payload_input) > 0) { + } else if (i_stream_have_bytes_left(resp->payload_input)) { /* output is blocking */ conn->output_locked = TRUE; o_stream_set_flush_pending(output, TRUE); @@ -646,9 +646,10 @@ static int http_server_response_send_real(struct http_server_response *resp, o_stream_ref(output); o_stream_cork(output); if (o_stream_sendv(output, iov, N_ELEMENTS(iov)) < 0) { - if (errno != EPIPE && errno != ECONNRESET) { - *error_r = t_strdup_printf("write(%s) failed: %m", - o_stream_get_name(output)); + if (output->stream_errno != EPIPE && + output->stream_errno != ECONNRESET) { + *error_r = t_strdup_printf("write(%s) failed: %s", + o_stream_get_name(output), o_stream_get_error(output)); } ret = -1; } diff --git a/src/lib-http/http-server.c b/src/lib-http/http-server.c index 7f9f28df4fc..8a3d4a99753 100644 --- a/src/lib-http/http-server.c +++ b/src/lib-http/http-server.c @@ -34,6 +34,8 @@ struct http_server *http_server_init(const struct http_server_settings *set) server->set.max_pipelined_requests = (set->max_pipelined_requests > 0 ? set->max_pipelined_requests : 1); server->set.request_limits = set->request_limits; + server->set.socket_send_buffer_size = set->socket_send_buffer_size; + server->set.socket_recv_buffer_size = set->socket_recv_buffer_size; server->set.debug = set->debug; server->conn_list = http_server_connection_list_init(); diff --git a/src/lib-http/http-server.h b/src/lib-http/http-server.h index b90a3967312..5d87f0fcdd2 100644 --- a/src/lib-http/http-server.h +++ b/src/lib-http/http-server.h @@ -24,6 +24,12 @@ struct http_server_settings { /* request limits */ struct http_request_limits request_limits; + /* the kernel send/receive buffer sizes used for the connection sockets. + Configuring this is mainly useful for the test suite. The kernel + defaults are used when these settings are 0. */ + size_t socket_send_buffer_size; + size_t socket_recv_buffer_size; + bool debug; }; diff --git a/src/lib-http/test-http-client-errors.c b/src/lib-http/test-http-client-errors.c new file mode 100644 index 00000000000..579fdf88a5b --- /dev/null +++ b/src/lib-http/test-http-client-errors.c @@ -0,0 +1,1932 @@ +/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "hostpid.h" +#include "ioloop.h" +#include "istream.h" +#include "ostream.h" +#include "time-util.h" +#include "connection.h" +#include "test-common.h" +#include "http-url.h" +#include "http-request.h" +#include "http-client.h" + +#include +#include +#include +#include + +/* + * Types + */ + +struct server_connection { + struct connection conn; + + pool_t pool; +}; + +typedef void (*test_server_init_t)(unsigned int index); +typedef void (*test_client_init_t) + (const struct http_client_settings *client_set); + +/* + * State + */ + +/* common */ +static struct ip_addr bind_ip; +static in_port_t *bind_ports = 0; +static struct ioloop *ioloop; +static bool debug = FALSE; + +/* server */ +static struct io *io_listen; +static int fd_listen = -1; +static pid_t *server_pids = NULL; +static unsigned int server_pids_count = 0; +static struct connection_list *server_conn_list; +static size_t server_read_max = 0; +static unsigned int server_index; +static void (*test_server_input)(struct server_connection *conn); + +/* client */ +static struct http_client *http_client = NULL; + +/* + * Forward declarations + */ + +/* server */ +static void test_server_run(unsigned int index); +static void +server_connection_deinit(struct server_connection **_conn); + +/* client */ +static void +test_client_defaults(struct http_client_settings *http_set); +static void test_client_deinit(void); + +/* test*/ +static void test_run_client_server( + const struct http_client_settings *client_set, + test_client_init_t client_test, + test_server_init_t server_test, + unsigned int server_tests_count) + ATTR_NULL(3); + +/* + * Host lookup failed + */ + +/* client */ + +struct _host_lookup_failed { + unsigned int count; +}; + +static void +test_client_host_lookup_failed_response( + const struct http_response *resp, + struct _host_lookup_failed *ctx) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED); + test_assert(resp->reason != NULL && *resp->reason != '\0'); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static void +test_client_host_lookup_failed(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _host_lookup_failed *ctx; + + ctx = i_new(struct _host_lookup_failed, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request(http_client, + "GET", "host.in-addr.arpa", "/host-lookup-failed.txt", + test_client_host_lookup_failed_response, ctx); + http_client_request_submit(hreq); + + hreq = http_client_request(http_client, + "GET", "host.in-addr.arpa", "/host-lookup-failed2.txt", + test_client_host_lookup_failed_response, ctx); + http_client_request_submit(hreq); +} + +/* test */ + +static void test_host_lookup_failed(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("host lookup failed"); + test_run_client_server(&http_client_set, + test_client_host_lookup_failed, + NULL, 0); + test_end(); +} + +/* + * Connection refused + */ + +/* server */ + +static void +test_server_connection_refused(unsigned int index ATTR_UNUSED) +{ + i_close_fd(&fd_listen); +} + +/* client */ + +struct _connection_refused { + unsigned int count; +}; + +static void +test_client_connection_refused_response( + const struct http_response *resp, + struct _connection_refused *ctx) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED); + test_assert(resp->reason != NULL && *resp->reason != '\0'); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static void +test_client_connection_refused(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _connection_refused *ctx; + + ctx = i_new(struct _connection_refused, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/connection-refused.txt", + test_client_connection_refused_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/connection-refused2.txt", + test_client_connection_refused_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); +} + +/* test */ + +static void test_connection_refused(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("connection refused"); + test_run_client_server(&http_client_set, + test_client_connection_refused, + test_server_connection_refused, 1); + test_end(); +} + +/* + * Connection timed out + */ + +/* client */ + +struct _connection_timed_out { + unsigned int count; +}; + +static void +test_client_connection_timed_out_response( + const struct http_response *resp, + struct _connection_timed_out *ctx) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED); + test_assert(resp->reason != NULL && *resp->reason != '\0'); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static void +test_client_connection_timed_out( + const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _connection_timed_out *ctx; + + ctx = i_new(struct _connection_timed_out, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request(http_client, + "GET", "192.168.0.0", "/connection-timed-out.txt", + test_client_connection_timed_out_response, ctx); + http_client_request_submit(hreq); + + hreq = http_client_request(http_client, + "GET", "192.168.0.0", "/connection-timed-out2.txt", + test_client_connection_timed_out_response, ctx); + http_client_request_submit(hreq); +} + +/* test */ + +static void test_connection_timed_out(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.connect_timeout_msecs = 1000; + http_client_set.max_attempts = 1; + + test_begin("connection timed out"); + test_run_client_server(&http_client_set, + test_client_connection_timed_out, + NULL, 0); + test_end(); +} + +/* + * Invalid redirect + */ + +/* server */ + +/* -> not accepted */ + +static void +test_invalid_redirect_input1(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 302 Redirect\r\n" + "Location: http://localhost:4444\r\n" + "\r\n"; + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_invalid_redirect1(unsigned int index) +{ + test_server_input = test_invalid_redirect_input1; + test_server_run(index); +} + +/* -> bad location */ + +static void +test_invalid_redirect_input2(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 302 Redirect\r\n" + "Location: unix:/var/run/dovecot/auth-master\r\n" + "\r\n"; + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_invalid_redirect2(unsigned int index) +{ + test_server_input = test_invalid_redirect_input2; + test_server_run(index); +} + +/* -> too many */ + +static void +test_invalid_redirect_input3(struct server_connection *conn) +{ + string_t *resp; + + resp = t_str_new(512); + str_printfa(resp, + "HTTP/1.1 302 Redirect\r\n" + "Location: http://%s:%u/friep.txt\r\n" + "\r\n", + net_ip2addr(&bind_ip), bind_ports[server_index+1]); + o_stream_nsend(conn->conn.output, + str_data(resp), str_len(resp)); + server_connection_deinit(&conn); +} + +static void test_server_invalid_redirect3(unsigned int index) +{ + test_server_input = test_invalid_redirect_input3; + test_server_run(index); +} + +/* client */ + +static void +test_client_invalid_redirect_response( + const struct http_response *resp, + void *context ATTR_UNUSED) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT); + test_assert(resp->reason != NULL && *resp->reason != '\0'); + io_loop_stop(ioloop); +} + +static void +test_client_invalid_redirect(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + + http_client = http_client_init(client_set); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/invalid-redirect.txt", + test_client_invalid_redirect_response, NULL); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); +} + +/* test */ + +static void test_invalid_redirect(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("invalid redirect: not accepted"); + http_client_set.max_redirects = 0; + test_run_client_server(&http_client_set, + test_client_invalid_redirect, + test_server_invalid_redirect1, 1); + test_end(); + + test_begin("invalid redirect: bad location"); + http_client_set.max_redirects = 1; + test_run_client_server(&http_client_set, + test_client_invalid_redirect, + test_server_invalid_redirect2, 1); + test_end(); + + test_begin("invalid redirect: too many"); + http_client_set.max_redirects = 1; + test_run_client_server(&http_client_set, + test_client_invalid_redirect, + test_server_invalid_redirect3, 3); + test_end(); +} + +/* + * Unseekable redirect + */ + +/* server */ + +static void +test_unseekable_redirect_input(struct server_connection *conn) +{ + string_t *resp; + + resp = t_str_new(512); + str_printfa(resp, + "HTTP/1.1 302 Redirect\r\n" + "Location: http://%s:%u/frml.txt\r\n" + "\r\n", + net_ip2addr(&bind_ip), bind_ports[server_index+1]); + o_stream_nsend(conn->conn.output, + str_data(resp), str_len(resp)); + server_connection_deinit(&conn); +} + +static void test_server_unseekable_redirect(unsigned int index) +{ + test_server_input = test_unseekable_redirect_input; + test_server_run(index); +} + +/* client */ + +static void +test_client_unseekable_redirect_response( + const struct http_response *resp, + void *context ATTR_UNUSED) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_ABORTED); + test_assert(resp->reason != NULL && *resp->reason != '\0'); + io_loop_stop(ioloop); +} + +static void +test_client_unseekable_redirect(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct istream *input; + + http_client = http_client_init(client_set); + + input = i_stream_create_from_data("FROP", 4); + input->seekable = FALSE; + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/unseekable-redirect.txt", + test_client_unseekable_redirect_response, NULL); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, FALSE); + http_client_request_submit(hreq); + + i_stream_unref(&input); +} + +/* test */ + +static void test_unseekable_redirect(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.max_redirects = 1; + + test_begin("unseekable redirect"); + test_run_client_server(&http_client_set, + test_client_unseekable_redirect, + test_server_unseekable_redirect, 2); + test_end(); +} + +/* + * Unseekable retry + */ + +/* server */ + +static void +test_unseekable_retry_input(struct server_connection *conn) +{ + server_connection_deinit(&conn); +} + +static void test_server_unseekable_retry(unsigned int index) +{ + test_server_input = test_unseekable_retry_input; + test_server_run(index); +} + +/* client */ + +static void +test_client_unseekable_retry_response( + const struct http_response *resp, + void *context ATTR_UNUSED) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_ABORTED); + test_assert(resp->reason != NULL && *resp->reason != '\0'); + io_loop_stop(ioloop); +} + +static void +test_client_unseekable_retry(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct istream *input; + + http_client = http_client_init(client_set); + + input = i_stream_create_from_data("FROP", 4); + input->seekable = FALSE; + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/unseekable-retry.txt", + test_client_unseekable_retry_response, NULL); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, FALSE); + http_client_request_submit(hreq); + + i_stream_unref(&input); +} + +/* test */ + +static void test_unseekable_retry(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.max_attempts = 3; + + test_begin("unseekable retry"); + test_run_client_server(&http_client_set, + test_client_unseekable_retry, + test_server_unseekable_retry, 2); + test_end(); +} + +/* + * Broken payload + */ + +/* server */ + +static void +test_broken_payload_input(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 18\r\n" + "\r\n" + "Everything is OK\r\n"; + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_broken_payload(unsigned int index) +{ + test_server_input = test_broken_payload_input; + test_server_run(index); +} + +/* client */ + +static void +test_client_broken_payload_response( + const struct http_response *resp, + void *context ATTR_UNUSED) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD); + test_assert(resp->reason != NULL && *resp->reason != '\0'); + io_loop_stop(ioloop); +} + +static void +test_client_broken_payload(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct istream *input; + + test_expect_errors(1); + + http_client = http_client_init(client_set); + + input = i_stream_create_error_str(EIO, "Moehahahaha!!"); + i_stream_set_name(input, "PURE EVIL"); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/broken-payload.txt", + test_client_broken_payload_response, NULL); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, FALSE); + http_client_request_submit(hreq); + + i_stream_unref(&input); +} + +/* test */ + +static void test_broken_payload(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("broken payload"); + test_run_client_server(&http_client_set, + test_client_broken_payload, + test_server_broken_payload, 1); + test_end(); +} + +/* + * Connection lost + */ + +/* server */ + +static void +test_connection_lost_input(struct server_connection *conn) +{ + ssize_t ret; + + if (server_read_max == 0) { + server_connection_deinit(&conn); + return; + } + + i_stream_set_max_buffer_size(conn->conn.input, server_read_max); + ret = i_stream_read(conn->conn.input); + if (ret == -2) { + server_connection_deinit(&conn); + return; + } + if (ret < 0) { + if (i_stream_is_eof(conn->conn.input)) + i_fatal("server: Client stream ended prematurely"); + else + i_fatal("server: Streem error: %s", + i_stream_get_error(conn->conn.input)); + } +} + +static void test_server_connection_lost(unsigned int index) +{ + test_server_input = test_connection_lost_input; + test_server_run(index); +} + +/* client */ + +struct _connection_lost_ctx { + unsigned int count; +}; + +struct _connection_lost_request_ctx { + struct _connection_lost_ctx *ctx; + struct http_client_request *req; +}; + +static void +test_client_connection_lost_response( + const struct http_response *resp, + struct _connection_lost_request_ctx *rctx) +{ + struct _connection_lost_ctx *ctx = rctx->ctx; + + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST); + test_assert(resp->reason != NULL && *resp->reason != '\0'); + + if (http_client_request_try_retry(rctx->req)) { + if (debug) + i_debug("retrying"); + return; + } + i_free(rctx); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static void +test_client_connection_lost(const struct http_client_settings *client_set) +{ + static const char payload[] = + "This is a useless payload that only serves as a means to give the " + "server the opportunity to close the connection before the payload " + "is finished."; + struct _connection_lost_ctx *ctx; + struct _connection_lost_request_ctx *rctx; + struct http_client_request *hreq; + struct istream *input; + + ctx = i_new(struct _connection_lost_ctx, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + input = i_stream_create_from_data(payload, sizeof(payload)-1); + + rctx = i_new(struct _connection_lost_request_ctx, 1); + rctx->ctx = ctx; + + rctx->req = hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/connection-lost.txt", + test_client_connection_lost_response, rctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, FALSE); + http_client_request_submit(hreq); + + rctx = i_new(struct _connection_lost_request_ctx, 1); + rctx->ctx = ctx; + + rctx->req = hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/connection-lost2.txt", + test_client_connection_lost_response, rctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + i_stream_unref(&input); +} + +/* test */ + +static void test_connection_lost(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + server_read_max = 0; + + test_begin("connection lost: one attempt"); + http_client_set.max_attempts = 1; + test_run_client_server(&http_client_set, + test_client_connection_lost, + test_server_connection_lost, 1); + test_end(); + + test_begin("connection lost: two attempts"); + http_client_set.max_attempts = 2; + test_run_client_server(&http_client_set, + test_client_connection_lost, + test_server_connection_lost, 1); + test_end(); + + test_begin("connection lost: three attempts"); + http_client_set.max_attempts = 3; + test_run_client_server(&http_client_set, + test_client_connection_lost, + test_server_connection_lost, 1); + test_end(); + + test_begin("connection lost: manual retry"); + http_client_set.max_attempts = 3; + http_client_set.no_auto_retry = TRUE; + test_run_client_server(&http_client_set, + test_client_connection_lost, + test_server_connection_lost, 1); + test_end(); +} + +/* + * Connection lost after 100-continue + */ + +/* server */ + +static void +test_connection_lost_100_input(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 100 Continue\r\n" + "\r\n"; + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_connection_lost_100(unsigned int index) +{ + test_server_input = test_connection_lost_100_input; + test_server_run(index); +} + +/* client */ + +struct _connection_lost_100_ctx { + unsigned int count; +}; + +static void +test_client_connection_lost_100_response( + const struct http_response *resp, + struct _connection_lost_100_ctx *ctx) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST); + test_assert(resp->reason != NULL && *resp->reason != '\0'); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static void +test_client_connection_lost_100( + const struct http_client_settings *client_set) +{ + static const char payload[] = + "This is a useless payload that only serves as a means to give the " + "server the opportunity to close the connection before the payload " + "is finished."; + struct _connection_lost_100_ctx *ctx; + struct http_client_request *hreq; + struct istream *input; + + ctx = i_new(struct _connection_lost_100_ctx, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + input = i_stream_create_from_data(payload, sizeof(payload)-1); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/connection-lost.txt", + test_client_connection_lost_100_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, TRUE); + http_client_request_submit(hreq); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/connection-lost2.txt", + test_client_connection_lost_100_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, TRUE); + http_client_request_submit(hreq); + + i_stream_unref(&input); +} + +/* test */ + +static void test_connection_lost_100(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + server_read_max = 0; + + test_begin("connection lost after 100-continue"); + http_client_set.max_attempts = 1; + test_run_client_server(&http_client_set, + test_client_connection_lost_100, + test_server_connection_lost_100, 1); + test_end(); +} + +/* + * Connection lost in sub-ioloop + */ + +/* server */ + +static void +test_connection_lost_sub_ioloop_input(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 0\r\n" + "\r\n"; + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_connection_lost_sub_ioloop(unsigned int index) +{ + test_server_input = test_connection_lost_sub_ioloop_input; + test_server_run(index); +} + +/* client */ + +struct _connection_lost_sub_ioloop_ctx { + unsigned int count; +}; + +static void +test_client_connection_lost_sub_ioloop_response2( + const struct http_response *resp, + struct ioloop *sub_ioloop) +{ + if (debug) + i_debug("SUB-RESPONSE: %u %s", resp->status, resp->reason); + io_loop_stop(sub_ioloop); +} + +static void +test_client_connection_lost_sub_ioloop_response( + const struct http_response *resp, + struct _connection_lost_sub_ioloop_ctx *ctx) +{ + struct http_client_request *hreq; + struct ioloop *sub_ioloop; + + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + test_assert(resp->status == 200); + test_assert(resp->reason != NULL && *resp->reason != '\0'); + + sub_ioloop = io_loop_create(); + http_client_switch_ioloop(http_client); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/connection-lost-sub-ioloop3.txt", + test_client_connection_lost_sub_ioloop_response2, sub_ioloop); + http_client_request_set_port(hreq, bind_ports[1]); + http_client_request_submit(hreq); + + io_loop_run(sub_ioloop); + io_loop_set_current(ioloop); + http_client_switch_ioloop(http_client); + io_loop_set_current(sub_ioloop); + io_loop_destroy(&sub_ioloop); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static void +test_client_connection_lost_sub_ioloop( + const struct http_client_settings *client_set) +{ + static const char payload[] = + "This is a useless payload that only serves as a means to give the " + "server the opportunity to close the connection before the payload " + "is finished."; + struct _connection_lost_sub_ioloop_ctx *ctx; + struct http_client_request *hreq; + struct istream *input; + + ctx = i_new(struct _connection_lost_sub_ioloop_ctx, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + input = i_stream_create_from_data(payload, sizeof(payload)-1); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/connection-lost-sub-ioloop.txt", + test_client_connection_lost_sub_ioloop_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, TRUE); + http_client_request_submit(hreq); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/connection-lost-sub-ioloop2.txt", + test_client_connection_lost_sub_ioloop_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_set_payload(hreq, input, TRUE); + http_client_request_submit(hreq); + + i_stream_unref(&input); +} + +/* test */ + +static void test_connection_lost_sub_ioloop(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + server_read_max = 0; + + test_begin("connection lost while running sub-ioloop"); + http_client_set.max_attempts = 1; + test_run_client_server(&http_client_set, + test_client_connection_lost_sub_ioloop, + test_server_connection_lost_sub_ioloop, 2); + test_end(); +} + +/* + * Early success + */ + +/* server */ + +static void +test_early_success_input(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 200 OK\r\n" + "Content-Length: 18\r\n" + "\r\n" + "Everything is OK\r\n"; + + usleep(200000); + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_early_success(unsigned int index) +{ + test_server_input = test_early_success_input; + test_server_run(index); +} + +/* client */ + +struct _early_success_ctx { + unsigned int count; +}; + +static void +test_client_early_success_response( + const struct http_response *resp, + struct _early_success_ctx *ctx) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + if (ctx->count == 2) + test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE); + else + test_assert(resp->status == 200); + test_assert(resp->reason != NULL && *resp->reason != '\0'); + if (--ctx->count == 0) { + io_loop_stop(ioloop); + i_free(ctx); + } +} + +static void +test_client_early_success(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _early_success_ctx *ctx; + string_t *payload; + unsigned int i; + + ctx = i_new(struct _early_success_ctx, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/early-success.txt", + test_client_early_success_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + + T_BEGIN { + payload = t_str_new(204800); + for (i = 0; i < 3200; i++) { + str_append(payload, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n"); + } + + http_client_request_set_payload_data + (hreq, str_data(payload), str_len(payload)); + } T_END; + http_client_request_submit(hreq); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/early-success2.txt", + test_client_early_success_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); +} + +/* test */ + +static void test_early_success(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.socket_send_buffer_size = 4096; + + test_begin("early succes"); + test_run_client_server(&http_client_set, + test_client_early_success, + test_server_early_success, 1); + test_end(); +} + +/* + * Bad response + */ + +/* server */ + +static void +test_bad_response_input(struct server_connection *conn) +{ + static const char *resp = + "HTTP/1.1 666 Really bad response\r\n" + "\r\n"; + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_bad_response(unsigned int index) +{ + test_server_input = test_bad_response_input; + test_server_run(index); +} + +/* client */ + +struct _bad_response_ctx { + unsigned int count; +}; + +static void +test_client_bad_response_response( + const struct http_response *resp, + struct _bad_response_ctx *ctx) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE); + test_assert(resp->reason != NULL && *resp->reason != '\0'); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static void +test_client_bad_response(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _bad_response_ctx *ctx; + + ctx = i_new(struct _bad_response_ctx, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/bad-response.txt", + test_client_bad_response_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/bad-response2.txt", + test_client_bad_response_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); +} + +/* test */ + +static void test_bad_response(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("bad response"); + test_run_client_server(&http_client_set, + test_client_bad_response, + test_server_bad_response, 1); + test_end(); +} + +/* + * Request timed out + */ + +/* server */ + +static void +test_request_timed_out_input(struct server_connection *conn ATTR_UNUSED) +{ + /* do nothing */ +} + +static void test_server_request_timed_out(unsigned int index) +{ + test_server_input = test_request_timed_out_input; + test_server_run(index); +} + +/* client */ + +struct _request_timed_out_ctx { + unsigned int count; +}; + +static void +test_client_request_timed_out_response( + const struct http_response *resp, + struct _request_timed_out_ctx *ctx) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT); + test_assert(resp->reason != NULL && *resp->reason != '\0'); + + if (--ctx->count == 0) { + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static void +test_client_request_timed_out(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _request_timed_out_ctx *ctx; + + ctx = i_new(struct _request_timed_out_ctx, 1); + ctx->count = 2; + + http_client = http_client_init(client_set); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/request-timed-out.txt", + test_client_request_timed_out_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/request-timed-out2.txt", + test_client_request_timed_out_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); +} + +/* test */ + +static void test_request_timed_out(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("request timed out: one attempt"); + http_client_set.request_timeout_msecs = 1000; + http_client_set.max_attempts = 1; + test_run_client_server(&http_client_set, + test_client_request_timed_out, + test_server_request_timed_out, 1); + test_end(); + + test_begin("request timed out: two attempts"); + http_client_set.request_timeout_msecs = 1000; + http_client_set.max_attempts = 1; + test_run_client_server(&http_client_set, + test_client_request_timed_out, + test_server_request_timed_out, 1); + test_end(); + + test_begin("request absolutely timed out"); + http_client_set.request_timeout_msecs = 0; + http_client_set.request_absolute_timeout_msecs = 2000; + http_client_set.max_attempts = 3; + test_run_client_server(&http_client_set, + test_client_request_timed_out, + test_server_request_timed_out, 1); + test_end(); + + test_begin("request double timed out"); + http_client_set.request_timeout_msecs = 500; + http_client_set.request_absolute_timeout_msecs = 2000; + http_client_set.max_attempts = 3; + test_run_client_server(&http_client_set, + test_client_request_timed_out, + test_server_request_timed_out, 1); + test_end(); +} + +/* + * Request aborted early + */ + +/* server */ + +static void +test_request_aborted_early_input(struct server_connection *conn ATTR_UNUSED) +{ + static const char *resp = + "HTTP/1.1 404 Not Found\r\n" + "\r\n"; + + /* wait one second to respon */ + sleep(1); + + /* respond */ + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_request_aborted_early(unsigned int index) +{ + test_server_input = test_request_aborted_early_input; + test_server_run(index); +} + +/* client */ + +struct _request_aborted_early_ctx { + struct http_client_request *req1, *req2; + struct timeout *to; +}; + +static void +test_client_request_aborted_early_response( + const struct http_response *resp, + struct _request_aborted_early_ctx *ctx ATTR_UNUSED) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + /* abort does not trigger callback */ + test_assert(FALSE); +} + +static void +test_client_request_aborted_early_timeout( + struct _request_aborted_early_ctx *ctx) +{ + timeout_remove(&ctx->to); + + if (ctx->req1 != NULL) { + /* abort early */ + http_client_request_abort(&ctx->req1); /* sent */ + http_client_request_abort(&ctx->req2); /* only queued */ + + /* wait a little for server to actually respond to an + already aborted request */ + ctx->to = timeout_add_short(1000, + test_client_request_aborted_early_timeout, ctx); + } else { + /* all done */ + i_free(ctx); + io_loop_stop(ioloop); + } +} + +static void +test_client_request_aborted_early( + const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _request_aborted_early_ctx *ctx; + + ctx = i_new(struct _request_aborted_early_ctx, 1); + + http_client = http_client_init(client_set); + + hreq = ctx->req1 = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/request-aborted-early.txt", + test_client_request_aborted_early_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = ctx->req2 = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/request-aborted-early2.txt", + test_client_request_aborted_early_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + ctx->to = timeout_add_short(500, + test_client_request_aborted_early_timeout, ctx); +} + +/* test */ + +static void test_request_aborted_early(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("request aborted early"); + test_run_client_server(&http_client_set, + test_client_request_aborted_early, + test_server_request_aborted_early, 1); + test_end(); +} + +/* + * Client deinit early + */ + +/* server */ + +static void +test_client_deinit_early_input(struct server_connection *conn ATTR_UNUSED) +{ + static const char *resp = + "HTTP/1.1 404 Not Found\r\n" + "\r\n"; + + /* wait one second to respon */ + sleep(1); + + /* respond */ + o_stream_nsend_str(conn->conn.output, resp); + server_connection_deinit(&conn); +} + +static void test_server_client_deinit_early(unsigned int index) +{ + test_server_input = test_client_deinit_early_input; + test_server_run(index); +} + +/* client */ + +struct _client_deinit_early_ctx { + struct timeout *to; +}; + +static void +test_client_client_deinit_early_response( + const struct http_response *resp, + struct _client_deinit_early_ctx *ctx ATTR_UNUSED) +{ + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + /* abort does not trigger callback */ + test_assert(FALSE); +} + +static void +test_client_client_deinit_early_timeout( + struct _client_deinit_early_ctx *ctx) +{ + timeout_remove(&ctx->to); + + /* deinit early */ + http_client_deinit(&http_client); + + /* all done */ + i_free(ctx); + io_loop_stop(ioloop); +} + +static void +test_client_client_deinit_early(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _client_deinit_early_ctx *ctx; + + ctx = i_new(struct _client_deinit_early_ctx, 1); + + http_client = http_client_init(client_set); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/client-deinit-early.txt", + test_client_client_deinit_early_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/client-deinit-early2.txt", + test_client_client_deinit_early_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); + + ctx->to = timeout_add_short(500, + test_client_client_deinit_early_timeout, ctx); +} + +/* test */ + +static void test_client_deinit_early(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + + test_begin("client deinit early"); + test_run_client_server(&http_client_set, + test_client_client_deinit_early, + test_server_client_deinit_early, 1); + test_end(); +} + +/* + * Retry with delay + */ + +/* server */ + +static void +test_retry_with_delay_input(struct server_connection *conn) +{ + string_t *resp; + + resp = t_str_new(512); + str_printfa(resp, + "HTTP/1.1 500 Internal Server Error\r\n" + "\r\n"); + o_stream_nsend(conn->conn.output, + str_data(resp), str_len(resp)); + server_connection_deinit(&conn); +} + +static void test_server_retry_with_delay(unsigned int index) +{ + test_server_input = test_retry_with_delay_input; + test_server_run(index); +} + +/* client */ + +struct _client_retry_with_delay_ctx { + struct http_client_request *req; + unsigned int retries; + struct timeval time; +}; + +static void +test_client_retry_with_delay_response( + const struct http_response *resp, + struct _client_retry_with_delay_ctx *ctx) +{ + int real_delay, exp_delay; + + if (debug) + i_debug("RESPONSE: %u %s", resp->status, resp->reason); + + test_assert(resp->status == 500); + test_assert(resp->reason != NULL && *resp->reason != '\0'); + + if (ctx->retries > 0) { + /* check delay */ + real_delay = timeval_diff_msecs(&ioloop_timeval, &ctx->time); + exp_delay = (1 << (ctx->retries-1)) * 50; + if (real_delay < exp_delay) { + i_fatal("Retry delay is too short %d < %d", + real_delay, exp_delay); + } + } + + http_client_request_delay_msecs(ctx->req, (1 << ctx->retries) * 50); + ctx->time = ioloop_timeval; + if (http_client_request_try_retry(ctx->req)) { + ctx->retries++; + if (debug) + i_debug("retrying"); + return; + } + + i_free(ctx); + io_loop_stop(ioloop); +} + +static void +test_client_retry_with_delay(const struct http_client_settings *client_set) +{ + struct http_client_request *hreq; + struct _client_retry_with_delay_ctx *ctx; + + ctx = i_new(struct _client_retry_with_delay_ctx, 1); + ctx->time = ioloop_timeval; + + http_client = http_client_init(client_set); + + ctx->req = hreq = http_client_request(http_client, + "GET", net_ip2addr(&bind_ip), "/retry-with-delay.txt", + test_client_retry_with_delay_response, ctx); + http_client_request_set_port(hreq, bind_ports[0]); + http_client_request_submit(hreq); +} + +/* test */ + +static void test_retry_with_delay(void) +{ + struct http_client_settings http_client_set; + + test_client_defaults(&http_client_set); + http_client_set.max_attempts = 3; + + test_begin("retry with delay"); + test_run_client_server(&http_client_set, + test_client_retry_with_delay, + test_server_retry_with_delay, 1); + test_end(); +} + +/* + * All tests + */ + +static void (*test_functions[])(void) = { + test_host_lookup_failed, + test_connection_refused, + test_connection_timed_out, + test_invalid_redirect, + test_unseekable_redirect, + test_unseekable_retry, + test_broken_payload, + test_connection_lost, + test_connection_lost_100, + test_connection_lost_sub_ioloop, + test_early_success, + test_bad_response, + test_request_timed_out, + test_request_aborted_early, + test_client_deinit_early, + test_retry_with_delay, + NULL +}; + +/* + * Test client + */ + +static void +test_client_defaults(struct http_client_settings *http_set) +{ + /* client settings */ + memset(http_set, 0, sizeof(*http_set)); + http_set->max_idle_time_msecs = 5*1000; + http_set->max_parallel_connections = 1; + http_set->max_pipelined_requests = 1; + http_set->max_redirects = 0; + http_set->max_attempts = 1; + http_set->debug = debug; +} + +static void test_client_deinit(void) +{ + if (http_client != NULL) + http_client_deinit(&http_client); + http_client = NULL; +} + +/* + * Test server + */ + +/* client connection */ + +static void +server_connection_input(struct connection *_conn) +{ + struct server_connection *conn = (struct server_connection *)_conn; + + test_server_input(conn); +} + +static void +server_connection_init(int fd) +{ + struct server_connection *conn; + pool_t pool; + + net_set_nonblock(fd, TRUE); + + pool = pool_alloconly_create("server connection", 256); + conn = p_new(pool, struct server_connection, 1); + conn->pool = pool; + + connection_init_server + (server_conn_list, &conn->conn, "server connection", fd, fd); +} + +static void +server_connection_deinit(struct server_connection **_conn) +{ + struct server_connection *conn = *_conn; + + *_conn = NULL; + + connection_deinit(&conn->conn); + pool_unref(&conn->pool); +} + +static void +server_connection_destroy(struct connection *_conn) +{ + struct server_connection *conn = + (struct server_connection *)_conn; + + server_connection_deinit(&conn); +} + +static void +server_connection_accept(void *context ATTR_UNUSED) +{ + int fd; + + /* accept new client */ + fd = net_accept(fd_listen, NULL, NULL); + if (fd == -1) + return; + if (fd == -2) { + i_fatal("test server: accept() failed: %m"); + } + + server_connection_init(fd); +} + +/* */ + +static struct connection_settings server_connection_set = { + .input_max_size = (size_t)-1, + .output_max_size = (size_t)-1, + .client = FALSE +}; + +static const struct connection_vfuncs server_connection_vfuncs = { + .destroy = server_connection_destroy, + .input = server_connection_input +}; + +static void test_server_run(unsigned int index) +{ + server_index = index; + + /* open server socket */ + io_listen = io_add(fd_listen, + IO_READ, server_connection_accept, (void *)NULL); + + server_conn_list = connection_list_init + (&server_connection_set, &server_connection_vfuncs); + + io_loop_run(ioloop); + + /* close server socket */ + io_remove(&io_listen); + + connection_list_deinit(&server_conn_list); +} + +/* + * Tests + */ + +static int test_open_server_fd(in_port_t *bind_port) +{ + int fd = net_listen(&bind_ip, bind_port, 128); + if (debug) + i_debug("server listening on %u", *bind_port); + if (fd == -1) { + i_fatal("listen(%s:%u) failed: %m", + net_ip2addr(&bind_ip), *bind_port); + } + return fd; +} + +static void test_servers_kill_all(void) +{ + unsigned int i; + + if (server_pids_count > 0) { + for (i = 0; i < server_pids_count; i++) { + if (server_pids[i] != (pid_t)-1) { + (void)kill(server_pids[i], SIGKILL); + (void)waitpid(server_pids[i], NULL, 0); + server_pids[i] = -1; + } + } + } + server_pids_count = 0; +} + +static void test_run_client_server( + const struct http_client_settings *client_set, + test_client_init_t client_test, + test_server_init_t server_test, + unsigned int server_tests_count) +{ + unsigned int i; + + server_pids = NULL; + server_pids_count = 0; + + if (server_tests_count > 0) { + int fds[server_tests_count]; + + bind_ports = i_new(in_port_t, server_tests_count); + + server_pids = i_new(pid_t, server_tests_count); + for (i = 0; i < server_tests_count; i++) + server_pids[i] = (pid_t)-1; + server_pids_count = server_tests_count; + + for (i = 0; i < server_tests_count; i++) + fds[i] = test_open_server_fd(&bind_ports[i]); + + for (i = 0; i < server_tests_count; i++) { + fd_listen = fds[i]; + if ((server_pids[i] = fork()) == (pid_t)-1) + i_fatal("fork() failed: %m"); + if (server_pids[i] == 0) { + server_pids[i] = (pid_t)-1; + server_pids_count = 0; + hostpid_init(); + if (debug) + i_debug("server[%d]: PID=%s", i+1, my_pid); + /* child: server */ + ioloop = io_loop_create(); + server_test(i); + io_loop_destroy(&ioloop); + if (fd_listen != -1) + i_close_fd(&fd_listen); + i_free(bind_ports); + i_free(server_pids); + /* wait for it to be killed; this way, valgrind will not + object to this process going away inelegantly. */ + sleep(60); + exit(1); + } + if (fd_listen != -1) + i_close_fd(&fd_listen); + } + if (debug) + i_debug("client: PID=%s", my_pid); + } + + /* parent: client */ + + usleep(100000); /* wait a little for server setup */ + + ioloop = io_loop_create(); + client_test(client_set); + io_loop_run(ioloop); + test_client_deinit(); + io_loop_destroy(&ioloop); + + test_servers_kill_all(); + i_free(server_pids); + i_free(bind_ports); +} + +/* + * Main + */ + +volatile sig_atomic_t terminating = 0; + +static void +test_signal_handler(int signo) +{ + if (terminating) + raise(signo); + terminating = 1; + + /* make sure we don't leave any pesky children alive */ + test_servers_kill_all(); + + (void)signal(signo, SIG_DFL); + raise(signo); +} + +static void test_atexit(void) +{ + test_servers_kill_all(); +} + +int main(int argc, char *argv[]) +{ + int c; + + atexit(test_atexit); + (void)signal(SIGCHLD, SIG_IGN); + (void)signal(SIGTERM, test_signal_handler); + (void)signal(SIGQUIT, test_signal_handler); + (void)signal(SIGINT, test_signal_handler); + (void)signal(SIGSEGV, test_signal_handler); + (void)signal(SIGABRT, test_signal_handler); + + while ((c = getopt(argc, argv, "D")) > 0) { + switch (c) { + case 'D': + debug = TRUE; + break; + default: + i_fatal("Usage: %s [-D]", argv[0]); + } + } + + /* listen on localhost */ + memset(&bind_ip, 0, sizeof(bind_ip)); + bind_ip.family = AF_INET; + bind_ip.u.ip4.s_addr = htonl(INADDR_LOOPBACK); + + test_run(test_functions); +} diff --git a/src/lib-http/test-http-client.c b/src/lib-http/test-http-client.c index 883cf52ddbb..3b3a26c156c 100644 --- a/src/lib-http/test-http-client.c +++ b/src/lib-http/test-http-client.c @@ -182,6 +182,7 @@ static void run_tests(struct http_client *http_client) http_client_request_set_ssl(http_req, TRUE); http_client_request_submit(http_req); http_client_request_abort(&http_req); + i_free(test_req); test_req = i_new(struct http_test_request, 1); http_req = http_client_request(http_client, diff --git a/src/lib-http/test-http-payload.c b/src/lib-http/test-http-payload.c index 6e85be8b04e..047ff993db7 100644 --- a/src/lib-http/test-http-payload.c +++ b/src/lib-http/test-http-payload.c @@ -31,10 +31,15 @@ static bool request_100_continue = FALSE; static size_t read_server_partial = 0; static size_t read_client_partial = 0; static unsigned int test_max_pending = 200; +static unsigned int client_ioloop_nesting = 0; static struct ip_addr bind_ip; static in_port_t bind_port = 0; static int fd_listen = -1; +static pid_t server_pid = (pid_t)-1; +static struct ioloop *ioloop_nested = NULL; +static unsigned ioloop_nested_first = 0; +static unsigned ioloop_nested_last = 0; /* * Test files @@ -582,6 +587,9 @@ static void test_server_deinit(void) */ struct test_client_request { + struct test_client_request *prev, *next; + struct http_client_request *hreq; + struct io *io; struct istream *payload; struct istream *file; @@ -589,8 +597,20 @@ struct test_client_request { }; static struct http_client *http_client; +static struct test_client_request *client_requests; static unsigned int client_files_first, client_files_last; +static struct test_client_request * +test_client_request_new(void) +{ + struct test_client_request *tcreq; + + tcreq = i_new(struct test_client_request, 1); + DLLIST_PREPEND(&client_requests, tcreq); + + return tcreq; +} + static void test_client_request_destroy(void *context) { @@ -603,25 +623,41 @@ test_client_request_destroy(void *context) i_stream_unref(&tcreq->payload); if (tcreq->file != NULL) i_stream_unref(&tcreq->file); + + DLLIST_REMOVE(&client_requests, tcreq); i_free(tcreq); } +static void +test_client_requests_switch_ioloop(void) +{ + struct test_client_request *tcreq; + + for (tcreq = client_requests; tcreq != NULL; + tcreq = tcreq->next) { + if (tcreq->io != NULL) + tcreq->io = io_loop_move_io(&tcreq->io); + if (tcreq->payload != NULL) + i_stream_switch_ioloop(tcreq->payload); + } +} + /* download */ static void test_client_download_continue(void); static void -test_client_download_finished(struct test_client_request *tcreq) +test_client_download_finished(unsigned int files_idx) { const char **paths; unsigned int count; paths = array_get_modifiable(&files, &count); - i_assert(tcreq->files_idx < count); + i_assert(files_idx < count); i_assert(client_files_first < count); - i_assert(paths[tcreq->files_idx] != NULL); + i_assert(paths[files_idx] != NULL); - paths[tcreq->files_idx] = NULL; + paths[files_idx] = NULL; test_client_download_continue(); } @@ -631,6 +667,7 @@ test_client_download_payload_input(struct test_client_request *tcreq) struct istream *payload = tcreq->payload; const unsigned char *pdata, *fdata; size_t psize, fsize, pleft; + unsigned int files_idx = tcreq->files_idx; off_t ret; /* read payload */ @@ -688,13 +725,13 @@ test_client_download_payload_input(struct test_client_request *tcreq) tcreq->files_idx); } - /* finished */ - test_client_download_finished(tcreq); - /* dereference payload stream; finishes the request */ tcreq->payload = NULL; io_remove(&tcreq->io); /* holds a reference too */ i_stream_unref(&payload); + + /* finished */ + test_client_download_finished(files_idx); } } @@ -740,7 +777,7 @@ test_client_download_response(const struct http_response *resp, path, resp->status, resp->reason); } i_stream_unref(&fstream); - test_client_download_finished(tcreq); + test_client_download_finished(tcreq->files_idx); return; } @@ -751,7 +788,7 @@ test_client_download_response(const struct http_response *resp, path, tcreq->files_idx); } i_stream_unref(&fstream); - test_client_download_finished(tcreq); + test_client_download_finished(tcreq->files_idx); return; } @@ -805,7 +842,7 @@ static void test_client_download_continue(void) client_files_last++) { const char *path = paths[client_files_last]; - tcreq = i_new(struct test_client_request, 1); + tcreq = test_client_request_new(); tcreq->files_idx = client_files_last; if (debug) { @@ -813,7 +850,7 @@ static void test_client_download_continue(void) "retrieving %s [%u]", path, tcreq->files_idx); } - hreq = http_client_request(http_client, + hreq = tcreq->hreq = http_client_request(http_client, "GET", net_ip2addr(&bind_ip), t_strconcat("/download/", path, NULL), test_client_download_response, tcreq); @@ -840,17 +877,17 @@ test_client_download(const struct http_client_settings *client_set) static void test_client_echo_continue(void); static void -test_client_echo_finished(struct test_client_request *tcreq) +test_client_echo_finished(unsigned int files_idx) { const char **paths; unsigned int count; paths = array_get_modifiable(&files, &count); - i_assert(tcreq->files_idx < count); + i_assert(files_idx < count); i_assert(client_files_first < count); - i_assert(paths[tcreq->files_idx] != NULL); + i_assert(paths[files_idx] != NULL); - paths[tcreq->files_idx] = NULL; + paths[files_idx] = NULL; test_client_echo_continue(); } @@ -860,6 +897,7 @@ test_client_echo_payload_input(struct test_client_request *tcreq) struct istream *payload = tcreq->payload; const unsigned char *pdata, *fdata; size_t psize, fsize, pleft; + unsigned int files_idx = tcreq->files_idx; off_t ret; /* read payload */ @@ -917,13 +955,13 @@ test_client_echo_payload_input(struct test_client_request *tcreq) tcreq->files_idx); } - /* finished */ - test_client_echo_finished(tcreq); - /* dereference payload stream; finishes the request */ tcreq->payload = NULL; io_remove(&tcreq->io); /* holds a reference too */ i_stream_unref(&payload); + + /* finished */ + test_client_echo_finished(files_idx); } } @@ -981,7 +1019,7 @@ test_client_echo_response(const struct http_response *resp, path, tcreq->files_idx); } i_stream_unref(&fstream); - test_client_echo_finished(tcreq); + test_client_echo_finished(tcreq->files_idx); return; } @@ -1000,7 +1038,7 @@ static void test_client_echo_continue(void) struct test_client_request *tcreq; struct http_client_request *hreq; const char **paths; - unsigned int count; + unsigned int count, first_submitted; paths = array_get_modifiable(&files, &count); @@ -1009,7 +1047,7 @@ static void test_client_echo_continue(void) i_assert(client_files_first <= client_files_last); for (; client_files_first < client_files_last && - paths[client_files_first] == NULL; client_files_first++) + paths[client_files_first] == NULL; client_files_first++); if (debug) { i_debug("test client: echo: " @@ -1019,7 +1057,9 @@ static void test_client_echo_continue(void) if (debug && client_files_first < count) { const char *path = paths[client_files_first]; i_debug("test client: echo: " - "next blocking: %s", (path == NULL ? "none" : path)); + "next blocking: %s [%d]", + (path == NULL ? "none" : path), + client_files_first); } if (client_files_first >= count) { @@ -1027,6 +1067,7 @@ static void test_client_echo_continue(void) return; } + first_submitted = client_files_last; for (; client_files_last < count && (client_files_last - client_files_first) < test_max_pending; client_files_last++) { @@ -1050,10 +1091,10 @@ static void test_client_echo_continue(void) path, client_files_last); } - tcreq = i_new(struct test_client_request, 1); + tcreq = test_client_request_new(); tcreq->files_idx = client_files_last; - hreq = http_client_request(http_client, + hreq = tcreq->hreq = http_client_request(http_client, "PUT", net_ip2addr(&bind_ip), t_strconcat("/echo/", path, NULL), test_client_echo_response, tcreq); @@ -1066,6 +1107,60 @@ static void test_client_echo_continue(void) i_stream_unref(&fstream); } + + /* run nested ioloop (if requested) if new requests cross a nesting + boundary */ + if (ioloop_nested != NULL) { + unsigned int i; + + i_assert(ioloop_nested_first <= count); + i_assert(ioloop_nested_last <= count); + for (i = ioloop_nested_first; i < ioloop_nested_last; i++) { + if (paths[i] != NULL) { + if (debug) { + i_debug("test client: " + "not leaving ioloop [%u]", i); + } + break; + } + } + + if (i == ioloop_nested_last) + io_loop_stop(ioloop_nested); + } else if (client_ioloop_nesting > 0 && + ((client_files_last / client_ioloop_nesting) != + (first_submitted / client_ioloop_nesting)) ) { + struct ioloop *prev_ioloop = current_ioloop; + + ioloop_nested_first = first_submitted; + ioloop_nested_last = first_submitted + client_ioloop_nesting; + if (ioloop_nested_last > client_files_last) + ioloop_nested_last = client_files_last; + + if (debug) { + i_debug("test client: echo: entering ioloop for %u...%u", + ioloop_nested_first, ioloop_nested_last); + } + + ioloop_nested = io_loop_create(); + http_client_switch_ioloop(http_client); + test_client_requests_switch_ioloop(); + + io_loop_run(ioloop_nested); + + io_loop_set_current(prev_ioloop); + http_client_switch_ioloop(http_client); + test_client_requests_switch_ioloop(); + io_loop_set_current(ioloop_nested); + io_loop_destroy(&ioloop_nested); + ioloop_nested = NULL; + + if (debug) { + i_debug("test client: echo: leaving ioloop for %u...%u", + ioloop_nested_first, ioloop_nested_last); + } + ioloop_nested_first = ioloop_nested_last = 0; + } } static void @@ -1101,24 +1196,33 @@ static void test_open_server_fd(void) } } +static void test_server_kill(void) +{ + if (server_pid != (pid_t)-1) { + (void)kill(server_pid, SIGKILL); + (void)waitpid(server_pid, NULL, 0); + } + server_pid = (pid_t)-1; +} + static void test_run_client_server( const struct http_client_settings *client_set, const struct http_server_settings *server_set, void (*client_init)(const struct http_client_settings *client_set)) { struct ioloop *ioloop; - pid_t pid; - int status; test_open_server_fd(); - if ((pid = fork()) == (pid_t)-1) + if ((server_pid = fork()) == (pid_t)-1) i_fatal("fork() failed: %m"); - if (pid == 0) { + if (server_pid == 0) { + server_pid = (pid_t)-1; hostpid_init(); if (debug) i_debug("server: PID=%s", my_pid); /* child: server */ + ioloop_nested = FALSE; ioloop = io_loop_create(); test_server_init(server_set); io_loop_run(ioloop); @@ -1130,14 +1234,13 @@ static void test_run_client_server( i_debug("client: PID=%s", my_pid); i_close_fd(&fd_listen); /* parent: client */ + ioloop_nested = FALSE; ioloop = io_loop_create(); client_init(client_set); io_loop_run(ioloop); test_client_deinit(); io_loop_destroy(&ioloop); - - (void)kill(pid, SIGKILL); - (void)waitpid(pid, &status, 0); + test_server_kill(); } } @@ -1240,6 +1343,7 @@ static void test_download_server_nonblocking(void) blocking = FALSE; request_100_continue = FALSE; read_server_partial = 0; + client_ioloop_nesting = 0; test_run_sequential(test_client_download); test_run_pipeline(test_client_download); test_run_parallel(test_client_download); @@ -1252,6 +1356,7 @@ static void test_download_server_blocking(void) blocking = TRUE; request_100_continue = FALSE; read_server_partial = 0; + client_ioloop_nesting = 0; test_run_sequential(test_client_download); test_run_pipeline(test_client_download); test_run_parallel(test_client_download); @@ -1264,6 +1369,7 @@ static void test_echo_server_nonblocking(void) blocking = FALSE; request_100_continue = FALSE; read_server_partial = 0; + client_ioloop_nesting = 0; test_run_sequential(test_client_echo); test_run_pipeline(test_client_echo); test_run_parallel(test_client_echo); @@ -1276,6 +1382,7 @@ static void test_echo_server_blocking(void) blocking = TRUE; request_100_continue = FALSE; read_server_partial = 0; + client_ioloop_nesting = 0; test_run_sequential(test_client_echo); test_run_pipeline(test_client_echo); test_run_parallel(test_client_echo); @@ -1288,6 +1395,7 @@ static void test_echo_server_nonblocking_sync(void) blocking = FALSE; request_100_continue = TRUE; read_server_partial = 0; + client_ioloop_nesting = 0; test_run_sequential(test_client_echo); test_run_pipeline(test_client_echo); test_run_parallel(test_client_echo); @@ -1300,6 +1408,7 @@ static void test_echo_server_blocking_sync(void) blocking = TRUE; request_100_continue = TRUE; read_server_partial = 0; + client_ioloop_nesting = 0; test_run_sequential(test_client_echo); test_run_pipeline(test_client_echo); test_run_parallel(test_client_echo); @@ -1312,6 +1421,7 @@ static void test_echo_server_nonblocking_partial(void) blocking = FALSE; request_100_continue = FALSE; read_server_partial = 1024; + client_ioloop_nesting = 0; test_run_sequential(test_client_echo); test_run_pipeline(test_client_echo); test_run_parallel(test_client_echo); @@ -1330,6 +1440,7 @@ static void test_echo_server_blocking_partial(void) blocking = TRUE; request_100_continue = FALSE; read_server_partial = 1024; + client_ioloop_nesting = 0; test_run_sequential(test_client_echo); test_run_pipeline(test_client_echo); test_run_parallel(test_client_echo); @@ -1349,6 +1460,7 @@ static void test_download_client_partial(void) request_100_continue = FALSE; read_server_partial = 0; read_client_partial = 1024; + client_ioloop_nesting = 0; test_run_sequential(test_client_download); test_run_pipeline(test_client_download); test_run_parallel(test_client_download); @@ -1364,6 +1476,18 @@ static void test_download_client_partial(void) test_end(); } +static void test_download_client_nested_ioloop(void) +{ + test_begin("http payload echo (client nested ioloop)"); + blocking = FALSE; + request_100_continue = FALSE; + read_server_partial = 0; + read_client_partial = 0; + client_ioloop_nesting = 10; + test_run_parallel(test_client_echo); + test_end(); +} + static void (*test_functions[])(void) = { test_download_server_nonblocking, test_download_server_blocking, @@ -1374,6 +1498,7 @@ static void (*test_functions[])(void) = { test_echo_server_nonblocking_partial, test_echo_server_blocking_partial, test_download_client_partial, + test_download_client_nested_ioloop, NULL }; @@ -1381,8 +1506,49 @@ static void (*test_functions[])(void) = { * Main */ -int main(void) +volatile sig_atomic_t terminating = 0; + +static void +test_signal_handler(int signo) { + if (terminating) + raise(signo); + terminating = 1; + + /* make sure we don't leave any pesky children alive */ + test_server_kill(); + + (void)signal(signo, SIG_DFL); + raise(signo); +} + +static void test_atexit(void) +{ + test_server_kill(); +} + +int main(int argc, char *argv[]) +{ + int c; + + atexit(test_atexit); + (void)signal(SIGCHLD, SIG_IGN); + (void)signal(SIGTERM, test_signal_handler); + (void)signal(SIGQUIT, test_signal_handler); + (void)signal(SIGINT, test_signal_handler); + (void)signal(SIGSEGV, test_signal_handler); + (void)signal(SIGABRT, test_signal_handler); + + while ((c = getopt(argc, argv, "D")) > 0) { + switch (c) { + case 'D': + debug = TRUE; + break; + default: + i_fatal("Usage: %s [-D]", argv[0]); + } + } + /* listen on localhost */ memset(&bind_ip, 0, sizeof(bind_ip)); bind_ip.family = AF_INET; diff --git a/src/lib-http/test-http-server-errors.c b/src/lib-http/test-http-server-errors.c new file mode 100644 index 00000000000..9c432ce3c60 --- /dev/null +++ b/src/lib-http/test-http-server-errors.c @@ -0,0 +1,754 @@ +/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "hostpid.h" +#include "ioloop.h" +#include "istream.h" +#include "ostream.h" +#include "time-util.h" +#include "connection.h" +#include "test-common.h" +#include "http-url.h" +#include "http-request.h" +#include "http-server.h" + +#include +#include +#include +#include + +#define SERVER_MAX_TIMEOUT_MSECS 10*1000 + +/* + * Types + */ + +struct client_connection { + struct connection conn; + + pool_t pool; +}; + +typedef void (*test_server_init_t) + (const struct http_server_settings *server_set); +typedef void (*test_client_init_t)(unsigned int index); + +/* + * State + */ + +/* common */ +static struct ip_addr bind_ip; +static in_port_t bind_port = 0; +static struct ioloop *ioloop; +static bool debug = FALSE; + +/* server */ +static struct http_server *http_server = NULL; +static struct io *io_listen; +static int fd_listen = -1; +static void (*test_server_request)(struct http_server_request *req); + +/* client */ +static pid_t *client_pids = NULL; +static struct connection_list *client_conn_list; +static unsigned int client_pids_count = 0; +static unsigned int client_index; +static void (*test_client_connected)(struct client_connection *conn); +static void (*test_client_input)(struct client_connection *conn); + +/* + * Forward declarations + */ + +/* server */ +static void +test_server_defaults(struct http_server_settings *http_set); +static void +test_server_run(const struct http_server_settings *http_set); + +/* client */ +static void test_client_run(unsigned int index); + +/* test*/ +static void test_run_client_server( + const struct http_server_settings *server_set, + test_server_init_t server_test, + test_client_init_t client_test, + unsigned int client_tests_count) + ATTR_NULL(3); + +/* + * Slow request + */ + +/* client */ + +static void +test_slow_request_input(struct client_connection *conn ATTR_UNUSED) +{ + /* do nothing */ +} + +static void +test_slow_request_connected(struct client_connection *conn) +{ + (void)o_stream_send_str(conn->conn.output, + "GET / HTTP/1.1\r\n" + "Host: example.com\r\n" + "\r\n"); +} + +static void test_client_slow_request(unsigned int index) +{ + test_client_input = test_slow_request_input; + test_client_connected = test_slow_request_connected; + test_client_run(index); +} + +/* server */ + +struct _slow_request { + struct http_server_request *req; + struct timeout *to_delay; + unsigned int serviced:1; +}; + +static void +test_server_slow_request_destroyed(void *context) +{ + struct _slow_request *ctx = (struct _slow_request *)context; + test_assert(ctx->serviced); + if (ctx->to_delay != NULL) + timeout_remove(&ctx->to_delay); + i_free(ctx); + io_loop_stop(ioloop); +} + +static void +test_server_slow_request_delayed(struct _slow_request *ctx) +{ + struct http_server_response *resp; + struct http_server_request *req = ctx->req; + + resp = http_server_response_create(req, 200, "OK"); + http_server_response_submit(resp); + ctx->serviced = TRUE; + + http_server_request_unref(&req); +} + +static void +test_server_slow_request_request( + struct http_server_request *req) +{ + const struct http_request *hreq = + http_server_request_get(req); + struct _slow_request *ctx; + + if (debug) { + i_debug("REQUEST: %s %s HTTP/%u.%u", + hreq->method, hreq->target_raw, + hreq->version_major, hreq->version_minor); + } + + ctx = i_new(struct _slow_request, 1); + ctx->req = req; + + http_server_request_set_destroy_callback(req, + test_server_slow_request_destroyed, ctx); + + http_server_request_ref(req); + ctx->to_delay = timeout_add + (4000, test_server_slow_request_delayed, ctx); +} + +static void test_server_slow_request +(const struct http_server_settings *server_set) +{ + test_server_request = test_server_slow_request_request; + test_server_run(server_set); +} + +/* test */ + +static void test_slow_request(void) +{ + struct http_server_settings http_server_set; + + test_server_defaults(&http_server_set); + http_server_set.max_client_idle_time_msecs = 1000; + + test_begin("slow request"); + test_run_client_server(&http_server_set, + test_server_slow_request, + test_client_slow_request, 1); + test_end(); +} + +/* + * Hanging request payload + */ + +/* client */ + +static void +test_hanging_request_payload_connected(struct client_connection *conn) +{ + (void)o_stream_send_str(conn->conn.output, + "GET / HTTP/1.1\r\n" + "Host: example.com\r\n" + "Content-Length: 1000\r\n" + "\r\n" + "To be continued... or not"); +} + +static void test_client_hanging_request_payload(unsigned int index) +{ + test_client_connected = test_hanging_request_payload_connected; + test_client_run(index); +} + +/* server */ + +struct _hanging_request_payload { + struct http_server_request *req; + struct istream *payload_input; + struct io *io; + unsigned int serviced:1; +}; + +static void +test_server_hanging_request_payload_destroyed(void *context) +{ + struct _hanging_request_payload *ctx = + (struct _hanging_request_payload *)context; + test_assert(!ctx->serviced); + if (ctx->io != NULL) + io_remove(&ctx->io); + i_free(ctx); + io_loop_stop(ioloop); +} + +static void +test_server_hanging_request_payload_input(struct _hanging_request_payload *ctx) +{ + struct http_server_response *resp; + struct http_server_request *req = ctx->req; + const unsigned char *data; + size_t size; + int ret; + + if (debug) + i_debug("test server: got more payload"); + + while ((ret=i_stream_read_data + (ctx->payload_input, &data, &size, 0)) > 0) { + i_stream_skip(ctx->payload_input, size); + } + + if (ret == 0) + return; + if (ret < 0) { + if (debug) { + i_debug("test server: failed to read payload: %s", + i_stream_get_error(ctx->payload_input)); + } + i_stream_unref(&ctx->payload_input); + io_remove(&ctx->io); + http_server_request_fail_close(req, + 400, "Bad request"); + http_server_request_unref(&req); + return; + } + + resp = http_server_response_create(req, 200, "OK"); + http_server_response_submit(resp); + ctx->serviced = TRUE; + + i_stream_unref(&ctx->payload_input); + http_server_request_unref(&req); +} + +static void +test_server_hanging_request_payload_request( + struct http_server_request *req) +{ + const struct http_request *hreq = + http_server_request_get(req); + struct _hanging_request_payload *ctx; + + if (debug) { + i_debug("REQUEST: %s %s HTTP/%u.%u", + hreq->method, hreq->target_raw, + hreq->version_major, hreq->version_minor); + } + + ctx = i_new(struct _hanging_request_payload, 1); + ctx->req = req; + + http_server_request_set_destroy_callback(req, + test_server_hanging_request_payload_destroyed, ctx); + + ctx->payload_input = + http_server_request_get_payload_input(req, FALSE); + + http_server_request_ref(req); + ctx->io = io_add_istream(ctx->payload_input, + test_server_hanging_request_payload_input, ctx); + test_server_hanging_request_payload_input(ctx); +} + +static void test_server_hanging_request_payload +(const struct http_server_settings *server_set) +{ + test_server_request = test_server_hanging_request_payload_request; + test_server_run(server_set); +} + +/* test */ + +static void test_hanging_request_payload(void) +{ + struct http_server_settings http_server_set; + + test_server_defaults(&http_server_set); + http_server_set.max_client_idle_time_msecs = 1000; + + test_begin("hanging request payload"); + test_run_client_server(&http_server_set, + test_server_hanging_request_payload, + test_client_hanging_request_payload, 1); + test_end(); +} + +/* + * Hanging response payload + */ + +/* client */ + +static void +test_hanging_response_payload_connected(struct client_connection *conn) +{ + (void)o_stream_send_str(conn->conn.output, + "GET / HTTP/1.1\r\n" + "Host: example.com\r\n" + "Content-Length: 18\r\n" + "\r\n" + "Complete payload\r\n"); +} + +static void test_client_hanging_response_payload(unsigned int index) +{ + test_client_connected = test_hanging_response_payload_connected; + test_client_run(index); +} + +/* server */ + +struct _hanging_response_payload { + struct http_server_request *req; + struct istream *payload_input; + struct io *io; + unsigned int serviced:1; +}; + +static void +test_server_hanging_response_payload_destroyed(void *context) +{ + struct _hanging_response_payload *ctx = + (struct _hanging_response_payload *)context; + test_assert(!ctx->serviced); + if (ctx->io != NULL) + io_remove(&ctx->io); + i_free(ctx); + io_loop_stop(ioloop); +} + +static void +test_server_hanging_response_payload_request( + struct http_server_request *req) +{ + const struct http_request *hreq = + http_server_request_get(req); + struct http_server_response *resp; + struct _hanging_response_payload *ctx; + string_t *payload; + unsigned int i; + + if (debug) { + i_debug("REQUEST: %s %s HTTP/%u.%u", + hreq->method, hreq->target_raw, + hreq->version_major, hreq->version_minor); + } + + ctx = i_new(struct _hanging_response_payload, 1); + ctx->req = req; + + http_server_request_set_destroy_callback(req, + test_server_hanging_response_payload_destroyed, ctx); + + resp = http_server_response_create(req, 200, "OK"); + T_BEGIN { + payload = t_str_new(204800); + for (i = 0; i < 3200; i++) { + str_append(payload, + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n" + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n"); + } + + http_server_response_set_payload_data + (resp, str_data(payload), str_len(payload)); + } T_END; + http_server_response_submit(resp); +} + +static void test_server_hanging_response_payload +(const struct http_server_settings *server_set) +{ + test_server_request = test_server_hanging_response_payload_request; + test_server_run(server_set); +} + +/* test */ + +static void test_hanging_response_payload(void) +{ + struct http_server_settings http_server_set; + + test_server_defaults(&http_server_set); + http_server_set.socket_send_buffer_size = 4096; + http_server_set.max_client_idle_time_msecs = 1000; + + test_begin("hanging response payload"); + test_run_client_server(&http_server_set, + test_server_hanging_response_payload, + test_client_hanging_response_payload, 1); + test_end(); +} + +/* + * All tests + */ + +static void (*test_functions[])(void) = { + test_slow_request, + test_hanging_request_payload, + test_hanging_response_payload, + NULL +}; + +/* + * Test client + */ + +/* client connection */ + +static void +client_connection_input(struct connection *_conn) +{ + struct client_connection *conn = (struct client_connection *)_conn; + + if (test_client_input != NULL) + test_client_input(conn); +} + +static void +client_connection_connected(struct connection *_conn, bool success) +{ + struct client_connection *conn = (struct client_connection *)_conn; + + if (success && test_client_connected != NULL) + test_client_connected(conn); +} + +static void +client_connection_init(const struct ip_addr *ip, in_port_t port) +{ + struct client_connection *conn; + pool_t pool; + + pool = pool_alloconly_create("client connection", 256); + conn = p_new(pool, struct client_connection, 1); + conn->pool = pool; + + connection_init_client_ip(client_conn_list, + &conn->conn, ip, port); + (void)connection_client_connect(&conn->conn); +} + +static void +server_connection_deinit(struct client_connection **_conn) +{ + struct client_connection *conn = *_conn; + + *_conn = NULL; + + connection_deinit(&conn->conn); + pool_unref(&conn->pool); +} + +static void +client_connection_destroy(struct connection *_conn) +{ + struct client_connection *conn = + (struct client_connection *)_conn; + + server_connection_deinit(&conn); +} + +/* */ + +static struct connection_settings client_connection_set = { + .input_max_size = (size_t)-1, + .output_max_size = (size_t)-1, + .client = TRUE +}; + +static const struct connection_vfuncs client_connection_vfuncs = { + .destroy = client_connection_destroy, + .client_connected = client_connection_connected, + .input = client_connection_input +}; + +static void test_client_run(unsigned int index) +{ + client_index = index; + + if (debug) + i_debug("client connecting to %u", bind_port); + + client_conn_list = connection_list_init + (&client_connection_set, &client_connection_vfuncs); + + client_connection_init(&bind_ip, bind_port); + + io_loop_run(ioloop); + + /* close server socket */ + io_remove(&io_listen); + + connection_list_deinit(&client_conn_list); +} + +/* + * Test server + */ + +static void +test_server_defaults(struct http_server_settings *http_set) +{ + /* server settings */ + memset(http_set, 0, sizeof(*http_set)); + http_set->max_client_idle_time_msecs = 5*1000; + http_set->max_pipelined_requests = 1; + http_set->debug = debug; +} + +/* client connection */ + +static void +server_handle_request(void *context ATTR_UNUSED, + struct http_server_request *req) +{ + test_server_request(req); +} + +struct http_server_callbacks http_server_callbacks = { + .handle_request = server_handle_request +}; + +static void +server_connection_accept(void *context ATTR_UNUSED) +{ + int fd; + + /* accept new client */ + fd = net_accept(fd_listen, NULL, NULL); + if (fd == -1) + return; + if (fd == -2) { + i_fatal("test server: accept() failed: %m"); + } + + (void)http_server_connection_create(http_server, fd, fd, FALSE, + &http_server_callbacks, NULL); +} + +/* */ + +static void +test_server_timeout(void *context ATTR_UNUSED) +{ + i_fatal("Server timed out"); + io_loop_stop(ioloop); +} + +static void +test_server_run(const struct http_server_settings *http_set) +{ + struct timeout *to; + + to = timeout_add(SERVER_MAX_TIMEOUT_MSECS, + test_server_timeout, NULL); + + /* open server socket */ + io_listen = io_add(fd_listen, + IO_READ, server_connection_accept, (void *)NULL); + + http_server = http_server_init(http_set); + + io_loop_run(ioloop); + + /* close server socket */ + io_remove(&io_listen); + timeout_remove(&to); + + http_server_deinit(&http_server); +} + +/* + * Tests + */ + +static int test_open_server_fd(void) +{ + int fd = net_listen(&bind_ip, &bind_port, 128); + if (debug) + i_debug("server listening on %u", bind_port); + if (fd == -1) { + i_fatal("listen(%s:%u) failed: %m", + net_ip2addr(&bind_ip), bind_port); + } + return fd; +} + +static void test_clients_kill_all(void) +{ + unsigned int i; + + if (client_pids_count > 0) { + for (i = 0; i < client_pids_count; i++) { + if (client_pids[i] != (pid_t)-1) { + (void)kill(client_pids[i], SIGKILL); + (void)waitpid(client_pids[i], NULL, 0); + client_pids[i] = -1; + } + } + } + client_pids_count = 0; +} + +static void test_run_client_server( + const struct http_server_settings *server_set, + test_server_init_t server_test, + test_client_init_t client_test, + unsigned int client_tests_count) +{ + unsigned int i; + + client_pids = NULL; + client_pids_count = 0; + + fd_listen = test_open_server_fd(); + + if (client_tests_count > 0) { + client_pids = i_new(pid_t, client_tests_count); + for (i = 0; i < client_tests_count; i++) + client_pids[i] = (pid_t)-1; + client_pids_count = client_tests_count; + + for (i = 0; i < client_tests_count; i++) { + if ((client_pids[i] = fork()) == (pid_t)-1) + i_fatal("fork() failed: %m"); + if (client_pids[i] == 0) { + client_pids[i] = (pid_t)-1; + client_pids_count = 0; + hostpid_init(); + if (debug) + i_debug("client[%d]: PID=%s", i+1, my_pid); + /* child: client */ + usleep(100000); /* wait a little for server setup */ + i_close_fd(&fd_listen); + ioloop = io_loop_create(); + client_test(i); + io_loop_destroy(&ioloop); + i_free(client_pids); + /* wait for it to be killed; this way, valgrind will not + object to this process going away inelegantly. */ + sleep(60); + exit(1); + } + } + if (debug) + i_debug("server: PID=%s", my_pid); + } + + /* parent: server */ + + ioloop = io_loop_create(); + server_test(server_set); + io_loop_destroy(&ioloop); + + i_close_fd(&fd_listen); + + test_clients_kill_all(); + i_free(client_pids); +} + +/* + * Main + */ + +volatile sig_atomic_t terminating = 0; + +static void +test_signal_handler(int signo) +{ + if (terminating != 0) + raise(signo); + terminating = 1; + + /* make sure we don't leave any pesky children alive */ + test_clients_kill_all(); + + (void)signal(signo, SIG_DFL); + raise(signo); +} + +static void test_atexit(void) +{ + test_clients_kill_all(); +} + +int main(int argc, char *argv[]) +{ + int c; + + atexit(test_atexit); + (void)signal(SIGCHLD, SIG_IGN); + (void)signal(SIGTERM, test_signal_handler); + (void)signal(SIGQUIT, test_signal_handler); + (void)signal(SIGINT, test_signal_handler); + (void)signal(SIGSEGV, test_signal_handler); + (void)signal(SIGABRT, test_signal_handler); + + while ((c = getopt(argc, argv, "D")) > 0) { + switch (c) { + case 'D': + debug = TRUE; + break; + default: + i_fatal("Usage: %s [-D]", argv[0]); + } + } + + /* listen on localhost */ + memset(&bind_ip, 0, sizeof(bind_ip)); + bind_ip.family = AF_INET; + bind_ip.u.ip4.s_addr = htonl(INADDR_LOOPBACK); + + test_run(test_functions); +} diff --git a/src/lib-imap-storage/imap-msgpart.c b/src/lib-imap-storage/imap-msgpart.c index d8039e0caa6..7eeabb0fafe 100644 --- a/src/lib-imap-storage/imap-msgpart.c +++ b/src/lib-imap-storage/imap-msgpart.c @@ -166,13 +166,12 @@ imap_msgpart_get_header_fields(pool_t pool, const char *header_list, value = p_strdup(pool, t_str_ucase(value)); array_append(fields, &value, 1); } + /* istream-header-filter requires headers to be sorted */ + array_sort(fields, i_strcasecmp_p); } else { result = -1; } - /* istream-header-filter requires headers to be sorted */ - array_sort(fields, i_strcasecmp_p); - imap_parser_unref(&parser); i_stream_unref(&input); return result; @@ -184,6 +183,9 @@ imap_msgpart_parse_header_fields(struct imap_msgpart *msgpart, { ARRAY_TYPE(const_string) fields; + if (header_list[0] == ' ') + header_list++; + /* HEADER.FIELDS (list), HEADER.FIELDS.NOT (list) */ if (imap_msgpart_get_header_fields(msgpart->pool, header_list, &fields) < 0) @@ -267,14 +269,14 @@ int imap_msgpart_parse(const char *section, struct imap_msgpart **msgpart_r) if (section[6] == '\0') { msgpart->fetch_type = FETCH_HEADER; ret = 0; - } else if (strncmp(section, "HEADER.FIELDS ", 14) == 0) { - msgpart->fetch_type = FETCH_HEADER_FIELDS; - ret = imap_msgpart_parse_header_fields(msgpart, - section+14); - } else if (strncmp(section, "HEADER.FIELDS.NOT ", 18) == 0) { + } else if (strncmp(section, "HEADER.FIELDS.NOT", 17) == 0) { msgpart->fetch_type = FETCH_HEADER_FIELDS_NOT; ret = imap_msgpart_parse_header_fields(msgpart, - section+18); + section+17); + } else if (strncmp(section, "HEADER.FIELDS", 13) == 0) { + msgpart->fetch_type = FETCH_HEADER_FIELDS; + ret = imap_msgpart_parse_header_fields(msgpart, + section+13); } else { ret = -1; } @@ -393,8 +395,8 @@ imap_msgpart_get_partial_header(struct mail *mail, struct istream *mail_input, if (message_get_header_size(input, &hdr_size, &has_nuls) < 0) { errno = input->stream_errno; mail_storage_set_critical(mail->box->storage, - "read(%s) failed: %s", i_stream_get_name(mail_input), - i_stream_get_error(mail_input)); + "read(%s) failed: %s", i_stream_get_name(input), + i_stream_get_error(input)); i_stream_unref(&input); return -1; } diff --git a/src/lib-index/mail-cache-compress.c b/src/lib-index/mail-cache-compress.c index 1e82fe2ffa5..1d47b22d97d 100644 --- a/src/lib-index/mail-cache-compress.c +++ b/src/lib-index/mail-cache-compress.c @@ -363,6 +363,7 @@ mail_cache_compress_write(struct mail_cache *cache, } mail_cache_file_close(cache); + cache->opened = TRUE; cache->fd = fd; cache->st_ino = st.st_ino; cache->st_dev = st.st_dev; @@ -451,6 +452,11 @@ static int mail_cache_compress_locked(struct mail_cache *cache, return mail_cache_reopen(cache) < 0 ? -1 : 0; } + if (cache->fd != -1) { + /* make sure we have mapped it before reading. */ + if (mail_cache_map(cache, 0, 0, &data) < 0) + return -1; + } /* we want to recreate the cache. write it first to a temporary file */ fd = mail_index_create_tmp_file(cache->index, cache->filepath, &temp_path); diff --git a/src/lib-index/mail-cache-fields.c b/src/lib-index/mail-cache-fields.c index 98a030b9ca2..a46a2e400ea 100644 --- a/src/lib-index/mail-cache-fields.c +++ b/src/lib-index/mail-cache-fields.c @@ -105,7 +105,7 @@ void mail_cache_register_fields(struct mail_cache *cache, char *name; void *value; unsigned int new_idx; - unsigned int i, j; + unsigned int i, j, registered_count; new_idx = cache->fields_count; for (i = 0; i < fields_count; i++) { @@ -141,10 +141,11 @@ void mail_cache_register_fields(struct mail_cache *cache, cache->fields_count * sizeof(*cache->field_file_map), new_idx * sizeof(*cache->field_file_map)); + registered_count = cache->fields_count; for (i = 0; i < fields_count; i++) { unsigned int idx = fields[i].idx; - if (idx < cache->fields_count) + if (idx < registered_count) continue; /* new index - save it */ @@ -159,7 +160,9 @@ void mail_cache_register_fields(struct mail_cache *cache, hash_table_insert(cache->field_name_hash, name, POINTER_CAST(idx)); + registered_count++; } + i_assert(registered_count == new_idx); cache->fields_count = new_idx; } diff --git a/src/lib-index/mail-cache.c b/src/lib-index/mail-cache.c index 34c05c5d84b..b5b4876b0ed 100644 --- a/src/lib-index/mail-cache.c +++ b/src/lib-index/mail-cache.c @@ -77,6 +77,7 @@ void mail_cache_file_close(struct mail_cache *cache) mail_cache_set_syscall_error(cache, "close()"); cache->fd = -1; } + cache->opened = FALSE; } static void mail_cache_init_file_cache(struct mail_cache *cache) @@ -101,14 +102,17 @@ static int mail_cache_try_open(struct mail_cache *cache) { const void *data; + i_assert(!cache->opened); cache->opened = TRUE; if (MAIL_INDEX_IS_IN_MEMORY(cache->index)) return 0; + i_assert(cache->fd == -1); cache->fd = nfs_safe_open(cache->filepath, cache->index->readonly ? O_RDONLY : O_RDWR); if (cache->fd == -1) { + mail_cache_file_close(cache); if (errno == ENOENT) { cache->need_compress_file_seq = 0; return 0; @@ -120,8 +124,10 @@ static int mail_cache_try_open(struct mail_cache *cache) mail_cache_init_file_cache(cache); - if (mail_cache_map(cache, 0, 0, &data) < 0) + if (mail_cache_map(cache, 0, 0, &data) < 0) { + mail_cache_file_close(cache); return -1; + } return 1; } @@ -487,6 +493,8 @@ int mail_cache_open_and_verify(struct mail_cache *cache) { int ret; + if (cache->opened) + return 0; ret = mail_cache_try_open(cache); if (ret > 0) ret = mail_cache_header_fields_read(cache); diff --git a/src/lib-index/mail-index-fsck.c b/src/lib-index/mail-index-fsck.c index 850ae93c5d6..8b177885669 100644 --- a/src/lib-index/mail-index-fsck.c +++ b/src/lib-index/mail-index-fsck.c @@ -277,6 +277,7 @@ mail_index_fsck_extensions(struct mail_index *index, struct mail_index_map *map, "with invalid header size", i, name); hdr->header_size = offset; + buffer_set_used_size(map->hdr_copy_buf, hdr->header_size); break; } if (mail_index_map_ext_hdr_check(hdr, ext_hdr, name, @@ -420,6 +421,7 @@ mail_index_fsck_map(struct mail_index *index, struct mail_index_map *map) mail_index_fsck_records(index, map, &hdr); map->hdr = hdr; + i_assert(map->hdr_copy_buf->used == map->hdr.header_size); } int mail_index_fsck(struct mail_index *index) diff --git a/src/lib-index/mail-index-map-read.c b/src/lib-index/mail-index-map-read.c index 7081f8c0f36..e84c4b70c8d 100644 --- a/src/lib-index/mail-index-map-read.c +++ b/src/lib-index/mail-index-map-read.c @@ -238,6 +238,7 @@ mail_index_try_read_map(struct mail_index_map *map, mail_index_map_copy_hdr(map, hdr); map->hdr_base = map->hdr_copy_buf->data; + i_assert(map->hdr_copy_buf->used == map->hdr.header_size); return 1; } diff --git a/src/lib-index/mail-index-map.c b/src/lib-index/mail-index-map.c index de422cbabf3..f3b14f24877 100644 --- a/src/lib-index/mail-index-map.c +++ b/src/lib-index/mail-index-map.c @@ -327,7 +327,9 @@ static void mail_index_map_copy_records(struct mail_index_record_map *dest, size_t size; size = src->records_count * record_size; - dest->buffer = buffer_create_dynamic(default_pool, I_MIN(size, 1024)); + /* +1% so we have a bit of space to grow. useful for huge mailboxes. */ + dest->buffer = buffer_create_dynamic(default_pool, + size + I_MAX(size/100, 1024)); buffer_append(dest->buffer, src->records, size); dest->records = buffer_get_modifiable_data(dest->buffer, NULL); diff --git a/src/lib-index/mail-index-modseq.c b/src/lib-index/mail-index-modseq.c index 1d7c9ba3193..29ceb585d53 100644 --- a/src/lib-index/mail-index-modseq.c +++ b/src/lib-index/mail-index-modseq.c @@ -521,6 +521,7 @@ static void mail_index_modseq_update_header(struct mail_index_view *view, buffer_write(map->hdr_copy_buf, ext->hdr_offset, &new_modseq_hdr, sizeof(new_modseq_hdr)); map->hdr_base = map->hdr_copy_buf->data; + i_assert(map->hdr_copy_buf->used == map->hdr.header_size); } } diff --git a/src/lib-index/mail-index-strmap.c b/src/lib-index/mail-index-strmap.c index e55cc2b32f8..7360e259245 100644 --- a/src/lib-index/mail-index-strmap.c +++ b/src/lib-index/mail-index-strmap.c @@ -1018,8 +1018,8 @@ static int mail_index_strmap_recreate(struct mail_index_strmap_view *view) o_stream_cork(output); mail_index_strmap_recreate_write(view, output); if (o_stream_nfinish(output) < 0) { - mail_index_set_error(strmap->index, - "write(%s) failed: %m", temp_path); + mail_index_set_error(strmap->index, "write(%s) failed: %s", + temp_path, o_stream_get_error(output)); ret = -1; } o_stream_destroy(&output); diff --git a/src/lib-index/mail-index-sync-ext.c b/src/lib-index/mail-index-sync-ext.c index 51d5853bba6..e6d475ba8b8 100644 --- a/src/lib-index/mail-index-sync-ext.c +++ b/src/lib-index/mail-index-sync-ext.c @@ -259,48 +259,60 @@ sync_ext_resize(const struct mail_transaction_ext_intro *u, uint32_t ext_map_idx, struct mail_index_sync_map_ctx *ctx, bool no_shrink) { - struct mail_index_map *map = ctx->view->map; + struct mail_index_map *map; struct mail_index_ext *ext; struct mail_index_ext_header *ext_hdr; - uint32_t old_size, new_size, old_record_size; - bool modified = FALSE, reorder = FALSE; - + uint32_t old_padded_hdr_size, new_padded_hdr_size, old_record_size; + bool reorder = FALSE; + + ext = array_idx_modifiable(&ctx->view->map->extensions, ext_map_idx); + old_padded_hdr_size = MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size); + new_padded_hdr_size = MAIL_INDEX_HEADER_SIZE_ALIGN(u->hdr_size); + + if (ext->record_align != u->record_align || + ext->record_size != u->record_size) { + /* record changed */ + } else if (new_padded_hdr_size < old_padded_hdr_size) { + /* header is shrunk. do we allow? */ + if (no_shrink) + return; + } else if (ext->hdr_size == u->hdr_size) { + /* no changes */ + return; + } + /* something changed. get ourself a new map before we start changing + anything in it. */ + map = mail_index_sync_get_atomic_map(ctx); + /* ext was duplicated to the new map. */ ext = array_idx_modifiable(&map->extensions, ext_map_idx); - old_size = MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size); - new_size = MAIL_INDEX_HEADER_SIZE_ALIGN(u->hdr_size); - - if (new_size < old_size) { + if (new_padded_hdr_size < old_padded_hdr_size) { /* header shrank */ if (no_shrink) - new_size = old_size; + new_padded_hdr_size = old_padded_hdr_size; else { buffer_delete(map->hdr_copy_buf, - ext->hdr_offset + new_size, - old_size - new_size); + ext->hdr_offset + new_padded_hdr_size, + old_padded_hdr_size - new_padded_hdr_size); ext->hdr_size = u->hdr_size; - modified = TRUE; } - } else if (new_size > old_size) { + } else if (new_padded_hdr_size > old_padded_hdr_size) { /* header grown */ buffer_insert_zero(map->hdr_copy_buf, - ext->hdr_offset + old_size, - new_size - old_size); + ext->hdr_offset + old_padded_hdr_size, + new_padded_hdr_size - old_padded_hdr_size); ext->hdr_size = u->hdr_size; - modified = TRUE; } else { if (ext->hdr_size != u->hdr_size) { /* aligned sizes were the same, but the actual sizes had changed */ ext->hdr_size = u->hdr_size; - modified = TRUE; } } if (ext->record_align < u->record_align || (ext->record_align > u->record_align && !no_shrink)) { ext->record_align = u->record_align; - modified = TRUE; reorder = TRUE; } @@ -308,29 +320,25 @@ sync_ext_resize(const struct mail_transaction_ext_intro *u, if (ext->record_size < u->record_size || (ext->record_size > u->record_size && !no_shrink)) { ext->record_size = u->record_size; - modified = TRUE; reorder = TRUE; } - if (modified) { - i_assert((map->hdr_copy_buf->used % sizeof(uint64_t)) == 0); - map->hdr_base = map->hdr_copy_buf->data; - map->hdr.header_size = map->hdr_copy_buf->used; + i_assert((map->hdr_copy_buf->used % sizeof(uint64_t)) == 0); + map->hdr_base = map->hdr_copy_buf->data; + map->hdr.header_size = map->hdr_copy_buf->used; - ext_hdr = get_ext_header(map, ext); - ext_hdr->reset_id = ext->reset_id; - ext_hdr->hdr_size = ext->hdr_size; - ext_hdr->record_offset = ext->record_offset; - ext_hdr->record_size = ext->record_size; - ext_hdr->record_align = ext->record_align; - } else { - i_assert(map->hdr_base == map->hdr_copy_buf->data); - } + ext_hdr = get_ext_header(map, ext); + ext_hdr->reset_id = ext->reset_id; + ext_hdr->hdr_size = ext->hdr_size; + ext_hdr->record_offset = ext->record_offset; + ext_hdr->record_size = ext->record_size; + ext_hdr->record_align = ext->record_align; - if (new_size != old_size) { + if (new_padded_hdr_size != old_padded_hdr_size) { /* move all hdr_offset of all extensions after this one */ unsigned int i, count = array_count(&map->extensions); - ssize_t diff = (ssize_t)new_size - (ssize_t)old_size; + ssize_t diff = (ssize_t)new_padded_hdr_size - + (ssize_t)old_padded_hdr_size; ext = array_idx_modifiable(&map->extensions, 0); for (i = ext_map_idx + 1; i < count; i++) { @@ -339,13 +347,8 @@ sync_ext_resize(const struct mail_transaction_ext_intro *u, } } - if (reorder) { - map = mail_index_sync_get_atomic_map(ctx); + if (reorder) sync_ext_reorder(map, ext_map_idx, old_record_size); - } else if (modified) { - /* header size changed. recreate index file. */ - (void)mail_index_sync_get_atomic_map(ctx); - } } static bool @@ -475,6 +478,7 @@ int mail_index_sync_ext_intro(struct mail_index_sync_map_ctx *ctx, intro is corrupted */ ctx->cur_ext_map_idx = (uint32_t)-2; ctx->cur_ext_ignore = TRUE; + ctx->cur_ext_record_size = 0; if (u->ext_id != (uint32_t)-1 && (!array_is_created(&map->extensions) || @@ -534,6 +538,7 @@ int mail_index_sync_ext_intro(struct mail_index_sync_map_ctx *ctx, return -1; } + ctx->cur_ext_record_size = u->record_size; if (ext != NULL) { /* exists already */ if (u->reset_id == ext->reset_id) { @@ -569,6 +574,7 @@ static void mail_index_sync_ext_clear(struct mail_index_view *view, memset(buffer_get_space_unsafe(map->hdr_copy_buf, ext->hdr_offset, ext->hdr_size), 0, ext->hdr_size); map->hdr_base = map->hdr_copy_buf->data; + i_assert(map->hdr_copy_buf->used == map->hdr.header_size); for (seq = 1; seq <= view->map->rec_map->records_count; seq++) { rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq); @@ -639,6 +645,7 @@ int mail_index_sync_ext_hdr_update(struct mail_index_sync_map_ctx *ctx, buffer_write(map->hdr_copy_buf, ext->hdr_offset + offset, data, size); map->hdr_base = map->hdr_copy_buf->data; + i_assert(map->hdr_copy_buf->used == map->hdr.header_size); if (ext->index_idx == ctx->view->index->modseq_ext_id) mail_index_modseq_hdr_update(ctx->modseq_ctx); @@ -670,7 +677,7 @@ mail_index_sync_ext_rec_update(struct mail_index_sync_map_ctx *ctx, return 1; ext = array_idx(&view->map->extensions, ctx->cur_ext_map_idx); - i_assert(ext->record_offset + ext->record_size <= + i_assert(ext->record_offset + ctx->cur_ext_record_size <= view->map->hdr.record_size); rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq); @@ -691,7 +698,11 @@ mail_index_sync_ext_rec_update(struct mail_index_sync_map_ctx *ctx, } /* @UNSAFE */ - memcpy(old_data, u + 1, ext->record_size); + memcpy(old_data, u + 1, ctx->cur_ext_record_size); + if (ctx->cur_ext_record_size < ext->record_size) { + memset(PTR_OFFSET(old_data, ctx->cur_ext_record_size), 0, + ext->record_size - ctx->cur_ext_record_size); + } return 1; } @@ -719,7 +730,7 @@ mail_index_sync_ext_atomic_inc(struct mail_index_sync_map_ctx *ctx, return 1; ext = array_idx(&view->map->extensions, ctx->cur_ext_map_idx); - i_assert(ext->record_offset + ext->record_size <= + i_assert(ext->record_offset + ctx->cur_ext_record_size <= view->map->hdr.record_size); rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq); @@ -727,8 +738,8 @@ mail_index_sync_ext_atomic_inc(struct mail_index_sync_map_ctx *ctx, min_value = u->diff >= 0 ? 0 : (uint64_t)(-(int64_t)u->diff); - max_value = ext->record_size == 8 ? (uint64_t)-1 : - ((uint64_t)1 << (ext->record_size*8)) - 1; + max_value = ctx->cur_ext_record_size == 8 ? (uint64_t)-1 : + ((uint64_t)1 << (ctx->cur_ext_record_size*8)) - 1; if (u->diff <= 0) { /* skip */ } else if (max_value >= (uint32_t)u->diff) { @@ -740,7 +751,7 @@ mail_index_sync_ext_atomic_inc(struct mail_index_sync_map_ctx *ctx, return -1; } - switch (ext->record_size) { + switch (ctx->cur_ext_record_size) { case 1: { uint8_t *num = data; @@ -773,7 +784,7 @@ mail_index_sync_ext_atomic_inc(struct mail_index_sync_map_ctx *ctx, default: mail_index_sync_set_corrupted(ctx, "Extension record inc with invalid size=%u", - ext->record_size); + ctx->cur_ext_record_size); return -1; } if (orig_num < min_value) { diff --git a/src/lib-index/mail-index-sync-keywords.c b/src/lib-index/mail-index-sync-keywords.c index 8546310038d..7d5d0964f69 100644 --- a/src/lib-index/mail-index-sync-keywords.c +++ b/src/lib-index/mail-index-sync-keywords.c @@ -190,6 +190,7 @@ keywords_header_add(struct mail_index_sync_map_ctx *ctx, buffer_copy(map->hdr_copy_buf, ext->hdr_offset, buf, 0, buf->used); map->hdr_base = map->hdr_copy_buf->data; + i_assert(map->hdr_copy_buf->used == map->hdr.header_size); if (mail_index_map_parse_keywords(map) < 0) i_panic("Keyword update corrupted keywords header"); diff --git a/src/lib-index/mail-index-sync-private.h b/src/lib-index/mail-index-sync-private.h index 5ef8293208f..985c8da2b77 100644 --- a/src/lib-index/mail-index-sync-private.h +++ b/src/lib-index/mail-index-sync-private.h @@ -27,6 +27,7 @@ struct mail_index_sync_map_ctx { struct mail_index_view *view; struct mail_index_modseq_sync *modseq_ctx; uint32_t cur_ext_map_idx; + uint32_t cur_ext_record_size; uint32_t ext_intro_seq; uoff_t ext_intro_offset, ext_intro_end_offset; diff --git a/src/lib-index/mail-index-sync-update.c b/src/lib-index/mail-index-sync-update.c index bb35a29fb69..76cb8514c73 100644 --- a/src/lib-index/mail-index-sync-update.c +++ b/src/lib-index/mail-index-sync-update.c @@ -496,6 +496,7 @@ static int sync_header_update(const struct mail_transaction_header_update *u, buffer_write(map->hdr_copy_buf, u->offset, u + 1, u->size); map->hdr_base = map->hdr_copy_buf->data; + i_assert(map->hdr_copy_buf->used == map->hdr.header_size); /* @UNSAFE */ if ((uint32_t)(u->offset + u->size) <= sizeof(map->hdr)) { @@ -711,7 +712,6 @@ mail_index_sync_record_real(struct mail_index_sync_map_ctx *ctx, } case MAIL_TRANSACTION_EXT_REC_UPDATE: { const struct mail_transaction_ext_rec_update *rec; - const struct mail_index_ext *ext; unsigned int i, record_size; if (ctx->cur_ext_map_idx == (uint32_t)-1) { @@ -727,10 +727,8 @@ mail_index_sync_record_real(struct mail_index_sync_map_ctx *ctx, break; } - ext = array_idx(&ctx->view->map->extensions, - ctx->cur_ext_map_idx); /* the record is padded to 32bits in the transaction log */ - record_size = (sizeof(*rec) + ext->record_size + 3) & ~3; + record_size = (sizeof(*rec) + ctx->cur_ext_record_size + 3) & ~3; for (i = 0; i < hdr->size; i += record_size) { rec = CONST_PTR_OFFSET(data, i); @@ -957,6 +955,11 @@ int mail_index_sync_map(struct mail_index_map **_map, &reset, &reason); if (ret <= 0) { mail_index_view_close(&view); + if (force && ret < 0) { + /* if we failed because of a syscall error, make sure + we return a failure. */ + return -1; + } if (force && ret == 0) { /* the seq/offset is probably broken */ mail_index_set_error(index, "Index %s: Lost log for " diff --git a/src/lib-index/mail-index-transaction-export.c b/src/lib-index/mail-index-transaction-export.c index 1996c80c900..f846c891f1c 100644 --- a/src/lib-index/mail-index-transaction-export.c +++ b/src/lib-index/mail-index-transaction-export.c @@ -138,16 +138,14 @@ static void log_append_ext_intro(struct mail_index_export_context *ctx, /* generate a new intro structure */ intro = buffer_append_space_unsafe(buf, sizeof(*intro)); intro->ext_id = idx; + intro->record_size = rext->record_size; + intro->record_align = rext->record_align; if (idx == (uint32_t)-1) { intro->hdr_size = rext->hdr_size; - intro->record_size = rext->record_size; - intro->record_align = rext->record_align; intro->name_size = strlen(rext->name); } else { ext = array_idx(&t->view->index->map->extensions, idx); intro->hdr_size = ext->hdr_size; - intro->record_size = ext->record_size; - intro->record_align = ext->record_align; intro->name_size = 0; } intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK; diff --git a/src/lib-index/mail-index-transaction-finish.c b/src/lib-index/mail-index-transaction-finish.c index d399c33b9ab..dfd4da3c124 100644 --- a/src/lib-index/mail-index-transaction-finish.c +++ b/src/lib-index/mail-index-transaction-finish.c @@ -326,8 +326,8 @@ mail_index_transaction_convert_to_uids(struct mail_index_transaction *t) void mail_index_transaction_finish(struct mail_index_transaction *t) { if (array_is_created(&t->appends)) { - mail_index_update_day_headers(t); mail_index_transaction_sort_appends(t); + mail_index_update_day_headers(t); } mail_index_transaction_finish_flag_updates(t); diff --git a/src/lib-index/mail-index-transaction-update.c b/src/lib-index/mail-index-transaction-update.c index ac1cf62bc5a..784c7370b47 100644 --- a/src/lib-index/mail-index-transaction-update.c +++ b/src/lib-index/mail-index-transaction-update.c @@ -713,40 +713,84 @@ void mail_index_update_header(struct mail_index_transaction *t, } } +static void +mail_index_ext_rec_updates_resize(struct mail_index_transaction *t, + uint32_t ext_id, uint16_t new_record_size) +{ + ARRAY_TYPE(seq_array) *array, old_array; + unsigned int i; + + if (!array_is_created(&t->ext_rec_updates)) + return; + array = array_idx_modifiable(&t->ext_rec_updates, ext_id); + if (!array_is_created(array)) + return; + + old_array = *array; + memset(array, 0, sizeof(*array)); + mail_index_seq_array_alloc(array, new_record_size); + + /* copy the records' beginnings. leave the end zero-filled. */ + for (i = 0; i < array_count(&old_array); i++) { + const void *old_record = array_idx(&old_array, i); + + memcpy(array_append_space(array), old_record, + old_array.arr.element_size); + } + array_free(&old_array); +} + void mail_index_ext_resize(struct mail_index_transaction *t, uint32_t ext_id, uint32_t hdr_size, uint16_t record_size, uint16_t record_align) { + const struct mail_index_registered_ext *rext; + const struct mail_transaction_ext_intro *resizes; + unsigned int resizes_count; struct mail_transaction_ext_intro intro; - uint32_t old_record_size, old_record_align, old_header_size; + uint32_t old_record_size = 0, old_record_align, old_header_size; memset(&intro, 0, sizeof(intro)); + rext = array_idx(&t->view->index->extensions, ext_id); /* get ext_id from transaction's map if it's there */ if (!mail_index_map_get_ext_idx(t->view->map, ext_id, &intro.ext_id)) { /* have to create it */ - const struct mail_index_registered_ext *rext; - intro.ext_id = (uint32_t)-1; - rext = array_idx(&t->view->index->extensions, ext_id); - old_record_size = rext->record_size; old_record_align = rext->record_align; old_header_size = rext->hdr_size; } else { const struct mail_index_ext *ext; ext = array_idx(&t->view->map->extensions, intro.ext_id); - old_record_size = ext->record_size; old_record_align = ext->record_align; old_header_size = ext->hdr_size; } - /* allow only header size changes if extension records have already - been changed in transaction */ - i_assert(!array_is_created(&t->ext_rec_updates) || - record_size == (uint16_t)-1 || - (old_record_size == record_size && - old_record_align == record_align)); + /* get the record size. if there are any existing record updates, + they're using the registered size, not the map's existing + record_size. */ + if (array_is_created(&t->ext_resizes)) + resizes = array_get(&t->ext_resizes, &resizes_count); + else { + resizes = NULL; + resizes_count = 0; + } + if (ext_id < resizes_count && resizes[ext_id].name_size != 0) { + /* already resized once. use the resized value. */ + old_record_size = resizes[ext_id].record_size; + } else { + /* use the registered values. */ + old_record_size = rext->record_size; + } + + if (record_size != old_record_size && record_size != (uint16_t)-1) { + /* if record_size grows, we'll just resize the existing + ext_rec_updates array. it's not possible to shrink + record_size without data loss. */ + i_assert(record_size > old_record_size); + mail_index_ext_rec_updates_resize(t, ext_id, record_size); + } t->log_ext_updates = TRUE; diff --git a/src/lib-index/mail-index-util.c b/src/lib-index/mail-index-util.c index b633539588f..659d622764d 100644 --- a/src/lib-index/mail-index-util.c +++ b/src/lib-index/mail-index-util.c @@ -116,6 +116,18 @@ bool mail_index_seq_array_lookup(const ARRAY_TYPE(seq_array) *array, mail_index_seq_record_cmp, idx_r); } +void mail_index_seq_array_alloc(ARRAY_TYPE(seq_array) *array, + size_t record_size) +{ + size_t aligned_record_size = (record_size + 3) & ~3; + + i_assert(!array_is_created(array)); + + array_create(array, default_pool, + sizeof(uint32_t) + aligned_record_size, + 1024 / (sizeof(uint32_t) + aligned_record_size)); +} + bool mail_index_seq_array_add(ARRAY_TYPE(seq_array) *array, uint32_t seq, const void *record, size_t record_size, void *old_record) @@ -126,11 +138,8 @@ bool mail_index_seq_array_add(ARRAY_TYPE(seq_array) *array, uint32_t seq, /* records need to be 32bit aligned */ aligned_record_size = (record_size + 3) & ~3; - if (!array_is_created(array)) { - array_create(array, default_pool, - sizeof(seq) + aligned_record_size, - 1024 / (sizeof(seq) + aligned_record_size)); - } + if (!array_is_created(array)) + mail_index_seq_array_alloc(array, record_size); i_assert(array->arr.element_size == sizeof(seq) + aligned_record_size); if (mail_index_seq_array_lookup(array, seq, &idx)) { diff --git a/src/lib-index/mail-index-util.h b/src/lib-index/mail-index-util.h index eeb17e94991..b61e16a23aa 100644 --- a/src/lib-index/mail-index-util.h +++ b/src/lib-index/mail-index-util.h @@ -13,6 +13,8 @@ int mail_index_unpack_num(const uint8_t **p, const uint8_t *end, bool mail_index_seq_array_lookup(const ARRAY_TYPE(seq_array) *array, uint32_t seq, unsigned int *idx_r); +void mail_index_seq_array_alloc(ARRAY_TYPE(seq_array) *array, + size_t record_size); bool mail_index_seq_array_add(ARRAY_TYPE(seq_array) *array, uint32_t seq, const void *record, size_t record_size, void *old_record) ATTR_NULL(5); diff --git a/src/lib-index/mail-index-view-sync.c b/src/lib-index/mail-index-view-sync.c index d7c664ef3b1..2911d9d04b5 100644 --- a/src/lib-index/mail-index-view-sync.c +++ b/src/lib-index/mail-index-view-sync.c @@ -428,7 +428,7 @@ view_sync_get_log_lost_changes(struct mail_index_view_sync_ctx *ctx, /* handle expunges and sync flags */ seqi = seqj = 1; - while (seqi < old_count && seqj < new_count) { + while (seqi <= old_count && seqj <= new_count) { old_rec = MAIL_INDEX_REC_AT_SEQ(old_map, seqi); new_rec = MAIL_INDEX_REC_AT_SEQ(new_map, seqj); if (old_rec->uid == new_rec->uid) { diff --git a/src/lib-index/test-mail-index-sync-ext.c b/src/lib-index/test-mail-index-sync-ext.c index 7a9f29d152e..b5c0a1170a7 100644 --- a/src/lib-index/test-mail-index-sync-ext.c +++ b/src/lib-index/test-mail-index-sync-ext.c @@ -63,7 +63,7 @@ static void test_mail_index_sync_ext_atomic_inc(void) u.uid = 1; #define TEST_ATOMIC(_type, _value, _diff, _ret) \ { _type *n = ptr; *n = _value; } \ - ext->record_size = sizeof(_type); \ + ctx.cur_ext_record_size = sizeof(_type); \ u.diff = _diff; \ test_assert(mail_index_sync_ext_atomic_inc(&ctx, &u) == _ret); @@ -81,7 +81,7 @@ static void test_mail_index_sync_ext_atomic_inc(void) TEST_ATOMIC_BLOCK(uint8_t, 255); TEST_ATOMIC_BLOCK(uint16_t, 65535); - ext->record_size = 5; + ctx.cur_ext_record_size = 5; u.diff = 0; test_assert(mail_index_sync_ext_atomic_inc(&ctx, &u) == -1); diff --git a/src/lib-lda/duplicate.c b/src/lib-lda/duplicate.c index b63dcbdca53..609cf32fbfb 100644 --- a/src/lib-lda/duplicate.c +++ b/src/lib-lda/duplicate.c @@ -318,7 +318,8 @@ void duplicate_flush(struct duplicate_context *ctx) hash_table_iterate_deinit(&iter); if (o_stream_nfinish(output) < 0) { - i_error("write(%s) failed: %m", file->path); + i_error("write(%s) failed: %s", file->path, + o_stream_get_error(output)); o_stream_unref(&output); duplicate_file_free(&ctx->file); return; @@ -342,7 +343,8 @@ struct duplicate_context *duplicate_init(struct mail_user *user) } ctx = i_new(struct duplicate_context, 1); - ctx->path = i_strconcat(home, "/"DUPLICATE_FNAME, NULL); + ctx->path = home == NULL ? NULL : + i_strconcat(home, "/"DUPLICATE_FNAME, NULL); ctx->dotlock_set = default_duplicate_dotlock_set; mail_set = mail_user_set_get_storage_set(user); diff --git a/src/lib-lda/lmtp-client.c b/src/lib-lda/lmtp-client.c index e47e0e8b6e8..239c6a6c46d 100644 --- a/src/lib-lda/lmtp-client.c +++ b/src/lib-lda/lmtp-client.c @@ -604,8 +604,8 @@ static void lmtp_client_input(struct lmtp_client *client) lmtp_client_fail(client, "501 5.5.4 Command reply line too long"); } else if (client->input->stream_errno != 0) { - errno = client->input->stream_errno; - i_error("lmtp client: read() failed: %m"); + i_error("lmtp client: read() failed: %s", + i_stream_get_error(client->input)); lmtp_client_fail(client, ERRSTR_TEMP_REMOTE_FAILURE " (read failure)"); } else if (client->input->eof) { diff --git a/src/lib-lda/smtp-client.c b/src/lib-lda/smtp-client.c index 28bb1eb3b22..131766ef038 100644 --- a/src/lib-lda/smtp-client.c +++ b/src/lib-lda/smtp-client.c @@ -265,14 +265,14 @@ smtp_client_send_flush(struct smtp_client *smtp_client, } if (o_stream_nfinish(smtp_client->output) < 0) { - *error_r = t_strdup_printf("write(%s) failed: %m", - smtp_client->temp_path); + *error_r = t_strdup_printf("write(%s) failed: %s", + smtp_client->temp_path, o_stream_get_error(smtp_client->output)); return -1; } if (o_stream_seek(smtp_client->output, 0) < 0) { - *error_r = t_strdup_printf("lseek(%s) failed: %m", - smtp_client->temp_path); + *error_r = t_strdup_printf("lseek(%s) failed: %s", + smtp_client->temp_path, o_stream_get_error(smtp_client->output)); return -1; } diff --git a/src/lib-ldap/Makefile.am b/src/lib-ldap/Makefile.am index fa8e213fb34..5407d1a6868 100644 --- a/src/lib-ldap/Makefile.am +++ b/src/lib-ldap/Makefile.am @@ -11,19 +11,21 @@ AM_CPPFLAGS = \ libdovecot_ldap_la_SOURCES = \ ldap-client.c \ ldap-connection.c \ + ldap-connection-pool.c \ ldap-iterator.c \ ldap-search.c \ ldap-compare.c \ ldap-entry.c -libdovecot_ldap_la_DEPENDENCIES = +libdovecot_ldap_la_DEPENDENCIES = ../lib/liblib.la libdovecot_ldap_la_LDFLAGS = -export-dynamic -libdovecot_ldap_la_LIBADD = $(LDAP_LIBS) +libdovecot_ldap_la_LIBADD = ../lib/liblib.la $(LDAP_LIBS) headers = \ ldap-client.h noinst_HEADERS = \ + ldap-connection-pool.h \ ldap-private.h pkginc_libdir=$(pkgincludedir) diff --git a/src/lib-ldap/ldap-client.c b/src/lib-ldap/ldap-client.c index d8b9743dbea..bb679f7543c 100644 --- a/src/lib-ldap/ldap-client.c +++ b/src/lib-ldap/ldap-client.c @@ -1,21 +1,31 @@ /* Copyright (c) 2016 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "ldap-connection-pool.h" #include "ldap-private.h" +/* Max number of ldap-connections that can be created. For now this is + unlimited since we're assuming our callers aren't calling us with many + different settings. */ +#define LDAP_CONN_POOL_MAX_CONNECTIONS UINT_MAX + struct ldap_client { - /* for now we support just a single connection, but this could be - extended to a connection pool. */ - struct ldap_connection *conn; + struct ldap_connection_list *list; }; +static struct ldap_connection_pool *ldap_conn_pool = NULL; + int ldap_client_init(const struct ldap_client_settings *set, struct ldap_client **client_r, const char **error_r) { struct ldap_client *client; + if (ldap_conn_pool == NULL) + ldap_conn_pool = ldap_connection_pool_init(LDAP_CONN_POOL_MAX_CONNECTIONS); + client = i_new(struct ldap_client, 1); - if (ldap_connection_init(client, set, &client->conn, error_r) < 0) { + if (ldap_connection_pool_get(ldap_conn_pool, client, set, + &client->list, error_r) < 0) { i_free(client); return -1; } @@ -29,13 +39,13 @@ void ldap_client_deinit(struct ldap_client **_client) *_client = NULL; - ldap_connection_deinit(&client->conn); + ldap_connection_pool_unref(ldap_conn_pool, &client->list); i_free(client); } void ldap_client_switch_ioloop(struct ldap_client *client) { - ldap_connection_switch_ioloop(client->conn); + ldap_connection_switch_ioloop(client->list->conn); } #undef ldap_search_start @@ -43,7 +53,9 @@ void ldap_search_start(struct ldap_client *client, const struct ldap_search_input *input, ldap_result_callback_t *callback, void *context) { - return ldap_connection_search_start(client->conn, input, callback, context); + /* FIXME: we could support multiple concurrent LDAP connections to + the same host. */ + ldap_connection_search_start(client->list->conn, input, callback, context); } #undef ldap_compare_start @@ -51,5 +63,12 @@ void ldap_compare_start(struct ldap_client *client, const struct ldap_compare_input *input, ldap_result_callback_t *callback, void *context) { - return ldap_connection_compare_start(client->conn, input, callback, context); + ldap_connection_compare_start(client->list->conn, input, callback, context); +} + +void ldap_clients_cleanup(void) +{ + if (ldap_conn_pool != NULL && + !ldap_connection_pool_have_references(ldap_conn_pool)) + ldap_connection_pool_deinit(&ldap_conn_pool); } diff --git a/src/lib-ldap/ldap-client.h b/src/lib-ldap/ldap-client.h index 5ce14dc6bf7..f0707c87fbe 100644 --- a/src/lib-ldap/ldap-client.h +++ b/src/lib-ldap/ldap-client.h @@ -19,6 +19,8 @@ struct ldap_entry; typedef void ldap_result_callback_t(struct ldap_result *result, void *context); struct ldap_client_settings { + /* NOTE: when adding here, remember to update + ldap_connection_have_settings() and ldap_connection_init() */ const char *uri; const char *bind_dn; const char *password; @@ -58,6 +60,11 @@ int ldap_client_init(const struct ldap_client_settings *set, void ldap_client_deinit(struct ldap_client **client); void ldap_client_switch_ioloop(struct ldap_client *client); +/* Deinitialize all pooled LDAP connections if there are no references left. + This allows freeing the memory at deinit, but still allows multiple + independent code parts to use lib-ldap and call this function. */ +void ldap_clients_cleanup(void); + void ldap_search_start(struct ldap_client *client, const struct ldap_search_input *input, ldap_result_callback_t *callback, diff --git a/src/lib-ldap/ldap-connection-pool.c b/src/lib-ldap/ldap-connection-pool.c new file mode 100644 index 00000000000..8b2d7a997f3 --- /dev/null +++ b/src/lib-ldap/ldap-connection-pool.c @@ -0,0 +1,113 @@ +/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "llist.h" +#include "ldap-private.h" +#include "ldap-connection-pool.h" + +struct ldap_connection_pool { + struct ldap_connection_list *conn_list; + unsigned int conn_count; + + unsigned int max_connections; +}; + +static void ldap_connection_list_remove(struct ldap_connection_pool *pool, + struct ldap_connection_list *list) +{ + DLLIST_REMOVE(&pool->conn_list, list); + pool->conn_count--; + + ldap_connection_deinit(&list->conn); + i_free(list); +} + +static void +ldap_connection_pool_shrink_to(struct ldap_connection_pool *pool, + unsigned int max_count) +{ + struct ldap_connection_list *list, *next; + + list = pool->conn_list; + for (; list != NULL && pool->conn_count > max_count; list = next) { + next = list->next; + if (list->refcount == 0) + ldap_connection_list_remove(pool, list); + } +} + +struct ldap_connection_pool * +ldap_connection_pool_init(unsigned int max_connections) +{ + struct ldap_connection_pool *pool; + + pool = i_new(struct ldap_connection_pool, 1); + pool->max_connections = max_connections; + return pool; +} + +void ldap_connection_pool_deinit(struct ldap_connection_pool **_pool) +{ + struct ldap_connection_pool *pool = *_pool; + + *_pool = NULL; + + ldap_connection_pool_shrink_to(pool, 0); + i_assert(pool->conn_list == NULL); + i_free(pool); +} + +int ldap_connection_pool_get(struct ldap_connection_pool *pool, + struct ldap_client *client, + const struct ldap_client_settings *set, + struct ldap_connection_list **list_r, + const char **error_r) +{ + struct ldap_connection_list *list; + struct ldap_connection *conn; + + for (list = pool->conn_list; list != NULL; list = list->next) { + if (ldap_connection_have_settings(list->conn, set)) { + list->refcount++; + *list_r = list; + return 0; + } + } + if (ldap_connection_init(client, set, &conn, error_r) < 0) + return -1; + + list = i_new(struct ldap_connection_list, 1); + list->conn = conn; + list->refcount++; + + DLLIST_PREPEND(&pool->conn_list, list); + pool->conn_count++; + + ldap_connection_pool_shrink_to(pool, pool->max_connections); + *list_r = list; + return 0; +} + +void ldap_connection_pool_unref(struct ldap_connection_pool *pool, + struct ldap_connection_list **_list) +{ + struct ldap_connection_list *list = *_list; + + *_list = NULL; + + i_assert(list->refcount > 0); + + if (--list->refcount == 0) + ldap_connection_pool_shrink_to(pool, pool->max_connections); +} + +bool ldap_connection_pool_have_references(struct ldap_connection_pool *pool) +{ + struct ldap_connection_list *list; + + for (list = pool->conn_list; list != NULL; list = list->next) { + if (list->refcount > 0) + return TRUE; + } + return FALSE; +} diff --git a/src/lib-ldap/ldap-connection-pool.h b/src/lib-ldap/ldap-connection-pool.h new file mode 100644 index 00000000000..00cf1654eac --- /dev/null +++ b/src/lib-ldap/ldap-connection-pool.h @@ -0,0 +1,27 @@ +#ifndef LDAP_CONNECTION_POOL_H +#define LDAP_CONNECTION_POOL_H + +struct ldap_client; +struct ldap_client_settings; + +struct ldap_connection_list { + struct ldap_connection_list *prev, *next; + struct ldap_connection *conn; + int refcount; +}; + +struct ldap_connection_pool * +ldap_connection_pool_init(unsigned int max_connections); +void ldap_connection_pool_deinit(struct ldap_connection_pool **_pool); +/* Returns TRUE if there are connections with refcount>0 */ +bool ldap_connection_pool_have_references(struct ldap_connection_pool *pool); + +int ldap_connection_pool_get(struct ldap_connection_pool *pool, + struct ldap_client *client, + const struct ldap_client_settings *set, + struct ldap_connection_list **list_r, + const char **error_r); +void ldap_connection_pool_unref(struct ldap_connection_pool *pool, + struct ldap_connection_list **list); + +#endif diff --git a/src/lib-ldap/ldap-connection.c b/src/lib-ldap/ldap-connection.c index 7b5caa091df..31a27cf5139 100644 --- a/src/lib-ldap/ldap-connection.c +++ b/src/lib-ldap/ldap-connection.c @@ -16,6 +16,8 @@ static void ldap_connection_request_destroy(struct ldap_op_queue_entry **req); static int ldap_connection_connect(struct ldap_connection *conn); +static +void ldap_connection_send_next(struct ldap_connection *conn); void ldap_connection_deinit(struct ldap_connection **_conn) { @@ -55,10 +57,12 @@ int ldap_connection_setup(struct ldap_connection *conn, const char **error_r) } ldap_set_option(conn->conn, LDAP_OPT_X_TLS, &opt); + ldap_set_option(conn->conn, LDAP_OPT_X_TLS_REQUIRE_CERT, &opt); +#ifdef LDAP_OPT_X_TLS_PROTOCOL_MIN /* refuse to connect to SSLv2 as it's completely insecure */ opt = LDAP_OPT_X_TLS_PROTOCOL_SSL3; ldap_set_option(conn->conn, LDAP_OPT_X_TLS_PROTOCOL_MIN, &opt); - +#endif opt = conn->set.timeout_secs; /* default timeout */ ldap_set_option(conn->conn, LDAP_OPT_TIMEOUT, &opt); @@ -84,9 +88,49 @@ int ldap_connection_setup(struct ldap_connection *conn, const char **error_r) ldap_set_option(conn->conn, LDAP_OPT_REFERRALS, 0); +#ifdef LDAP_OPT_X_TLS_NEWCTX + opt = 0; + ldap_set_option(conn->conn, LDAP_OPT_X_TLS_NEWCTX, &opt); +#endif + return 0; } +bool ldap_connection_have_settings(struct ldap_connection *conn, + const struct ldap_client_settings *set) +{ + const struct ldap_client_settings *conn_set = &conn->set; + + if (strcmp(conn_set->uri, set->uri) != 0) + return FALSE; + if (null_strcmp(conn_set->bind_dn, set->bind_dn) != 0) + return FALSE; + if (null_strcmp(conn_set->password, set->password) != 0) + return FALSE; + if (conn_set->timeout_secs != set->timeout_secs || + conn_set->max_idle_time_secs != set->max_idle_time_secs || + conn_set->debug != set->debug || + conn_set->require_ssl != set->require_ssl || + conn_set->start_tls != set->start_tls) + return FALSE; + + if (set->ssl_set == NULL || !set->start_tls) + return TRUE; + + /* check SSL settings */ + if (null_strcmp(conn->ssl_set.protocols, set->ssl_set->protocols) != 0) + return FALSE; + if (null_strcmp(conn->ssl_set.cipher_list, set->ssl_set->cipher_list) != 0) + return FALSE; + if (null_strcmp(conn->ssl_set.ca_file, set->ssl_set->ca_file) != 0) + return FALSE; + if (null_strcmp(conn->ssl_set.cert, set->ssl_set->cert) != 0) + return FALSE; + if (null_strcmp(conn->ssl_set.key, set->ssl_set->key) != 0) + return FALSE; + return TRUE; +} + int ldap_connection_init(struct ldap_client *client, const struct ldap_client_settings *set, struct ldap_connection **conn_r, const char **error_r) @@ -121,12 +165,15 @@ int ldap_connection_init(struct ldap_client *client, conn->ssl_set.crypto_device = NULL; if (set->ssl_set != NULL) { + /* keep in sync with ldap_connection_have_settings() */ + conn->set.ssl_set = &conn->ssl_set; conn->ssl_set.protocols = p_strdup(pool, set->ssl_set->protocols); conn->ssl_set.cipher_list = p_strdup(pool, set->ssl_set->cipher_list); conn->ssl_set.ca_file = p_strdup(pool, set->ssl_set->ca_file); conn->ssl_set.cert = p_strdup(pool, set->ssl_set->cert); conn->ssl_set.key = p_strdup(pool, set->ssl_set->key); } + i_assert(ldap_connection_have_settings(conn, set)); if (ldap_connection_setup(conn, error_r) < 0) { ldap_connection_deinit(&conn); @@ -298,14 +345,21 @@ ldap_connection_connect_parse(struct ldap_connection *conn, "ldap_start_tls(uri=%s) failed: %s", conn->set.uri, result_errmsg)); ldap_memfree(result_errmsg); - return LDAP_UNAVAILABLE; /* make sure it disconnects */ + return LDAP_INVALID_CREDENTIALS; /* make sure it disconnects */ } } else { ret = ldap_parse_extended_result(conn->conn, message, &retoid, NULL, 0); /* retoid can be NULL even if ret == 0 */ - if (ret == 0 && retoid != NULL && strcmp(retoid, LDAP_EXOP_START_TLS) == 0) { + if (ret == 0) { ret = ldap_install_tls(conn->conn); - } else if (ret == 0) ret = 2; /* make it fail on next if */ + if (ret != 0) { + // if this fails we have to abort + ldap_connection_result_failure(conn, req, ret, t_strdup_printf( + "ldap_start_tls(uri=%s) failed: %s", + conn->set.uri, ldap_err2string(ret))); + return LDAP_INVALID_CREDENTIALS; + } + } if (ret != LDAP_SUCCESS) { if (conn->set.require_ssl) { ldap_connection_result_failure(conn, req, ret, t_strdup_printf( @@ -383,6 +437,28 @@ void ldap_connection_abort_request(struct ldap_op_queue_entry *req) i_unreached(); } +static +void ldap_connection_abort_all_requests(struct ldap_connection *conn) +{ + struct ldap_result res; + memset(&res, 0, sizeof(res)); + res.openldap_ret = LDAP_TIMEOUT; + res.error_string = "Aborting LDAP requests due to failure"; + + unsigned int n = aqueue_count(conn->request_queue); + for (unsigned int i = 0; i < n; i++) { + struct ldap_op_queue_entry **reqp = + array_idx_modifiable(&(conn->request_array), + aqueue_idx(conn->request_queue, i)); + if ((*reqp)->to_abort != NULL) + timeout_remove(&(*reqp)->to_abort); + if ((*reqp)->result_callback != NULL) + (*reqp)->result_callback(&res, (*reqp)->result_callback_ctx); + ldap_connection_request_destroy(reqp); + } + aqueue_clear(conn->request_queue); +} + static int ldap_connect_next_message(struct ldap_connection *conn, struct ldap_op_queue_entry *req, bool *finished_r) @@ -478,7 +554,7 @@ int ldap_connection_connect(struct ldap_connection *conn) void ldap_connection_kill(struct ldap_connection *conn) { if (conn->io != NULL) - io_remove(&(conn->io)); + io_remove_closed(&(conn->io)); if (conn->to_disconnect != NULL) timeout_remove(&(conn->to_disconnect)); if (conn->to_reconnect != NULL) @@ -496,7 +572,7 @@ void ldap_connection_kill(struct ldap_connection *conn) } } if (conn->conn != NULL) { - ldap_destroy(conn->conn); + ldap_unbind_ext(conn->conn, NULL, NULL); ldap_memfree(conn->scred); } conn->conn = NULL; @@ -550,8 +626,6 @@ ldap_connection_handle_message(struct ldap_connection *conn, case LDAP_CONNECT_ERROR: #endif case LDAP_UNAVAILABLE: - ldap_connection_kill(conn); - /* fall through */ case LDAP_OPERATIONS_ERROR: case LDAP_BUSY: /* requeue */ @@ -559,6 +633,12 @@ ldap_connection_handle_message(struct ldap_connection *conn, ldap_connection_send_next(conn); finished = FALSE; break; + case LDAP_INVALID_CREDENTIALS: { + /* fail everything */ + ldap_connection_kill(conn); + ldap_connection_abort_all_requests(conn); + return 0; + } case LDAP_SIZELIMIT_EXCEEDED: case LDAP_TIMELIMIT_EXCEEDED: case LDAP_NO_SUCH_ATTRIBUTE: @@ -574,7 +654,6 @@ ldap_connection_handle_message(struct ldap_connection *conn, case LDAP_ALIAS_DEREF_PROBLEM: case LDAP_FILTER_ERROR: case LDAP_LOCAL_ERROR: - case LDAP_INVALID_CREDENTIALS: finished = TRUE; break; default: diff --git a/src/lib-ldap/ldap-private.h b/src/lib-ldap/ldap-private.h index e6004db8b23..fa724f40df9 100644 --- a/src/lib-ldap/ldap-private.h +++ b/src/lib-ldap/ldap-private.h @@ -108,6 +108,8 @@ int ldap_connection_init(struct ldap_client *client, struct ldap_connection **conn_r, const char **error_r); void ldap_connection_deinit(struct ldap_connection **_conn); void ldap_connection_switch_ioloop(struct ldap_connection *conn); +bool ldap_connection_have_settings(struct ldap_connection *conn, + const struct ldap_client_settings *set); void ldap_connection_search_start(struct ldap_connection *conn, const struct ldap_search_input *input, diff --git a/src/lib-ldap/ldap-search.c b/src/lib-ldap/ldap-search.c index 79e3cde785e..132534033e1 100644 --- a/src/lib-ldap/ldap-search.c +++ b/src/lib-ldap/ldap-search.c @@ -48,7 +48,7 @@ ldap_search_callback(struct ldap_connection *conn, ret = ldap_parse_result(conn->conn, message, &result_err, NULL, &result_errmsg, NULL, NULL, 0); if (ret == LDAP_NO_RESULTS_RETURNED) { - ret = LDAP_SUCCESS; + /*ret = LDAP_SUCCESS;*/ } else if (ret != LDAP_SUCCESS) { ldap_search_result_failure(req, ret, t_strdup_printf( "ldap_parse_result() failed for search: %s", ldap_err2string(ret))); diff --git a/src/lib-mail/istream-attachment-extractor.c b/src/lib-mail/istream-attachment-extractor.c index bc62146d24f..09a21ee7cc8 100644 --- a/src/lib-mail/istream-attachment-extractor.c +++ b/src/lib-mail/istream-attachment-extractor.c @@ -594,7 +594,7 @@ static int astream_read_next(struct attachment_istream *astream, bool *retry_r) *retry_r = FALSE; - if (stream->pos - stream->skip >= stream->max_buffer_size) + if (stream->pos - stream->skip >= i_stream_get_max_buffer_size(&stream->istream)) return -2; if (astream->failed) { diff --git a/src/lib-mail/istream-binary-converter.c b/src/lib-mail/istream-binary-converter.c index 87f2f419a13..f2564422f37 100644 --- a/src/lib-mail/istream-binary-converter.c +++ b/src/lib-mail/istream-binary-converter.c @@ -180,7 +180,7 @@ static ssize_t i_stream_binary_converter_read(struct istream_private *stream) struct message_block block; size_t old_size, new_size; - if (stream->pos - stream->skip >= stream->max_buffer_size) + if (stream->pos - stream->skip >= i_stream_get_max_buffer_size(&stream->istream)) return -2; old_size = stream->pos - stream->skip; diff --git a/src/lib-mail/istream-header-filter.c b/src/lib-mail/istream-header-filter.c index af36a43d8c0..f83c02c15d2 100644 --- a/src/lib-mail/istream-header-filter.c +++ b/src/lib-mail/istream-header-filter.c @@ -38,7 +38,11 @@ struct header_filter_istream { unsigned int add_missing_eoh:1; unsigned int end_body_with_lf:1; unsigned int last_lf_added:1; + unsigned int last_orig_crlf:1; + unsigned int last_added_newline:1; unsigned int eoh_not_matched:1; + unsigned int callbacks_called:1; + unsigned int prev_matched:1; }; header_filter_callback *null_header_filter_callback = NULL; @@ -133,6 +137,8 @@ static void add_eol(struct header_filter_istream *mstream, bool orig_crlf) buffer_append(mstream->hdr_buf, "\r\n", 2); else buffer_append_c(mstream->hdr_buf, '\n'); + mstream->last_orig_crlf = orig_crlf; + mstream->last_added_newline = TRUE; } static ssize_t hdr_stream_update_pos(struct header_filter_istream *mstream) @@ -151,8 +157,8 @@ static ssize_t read_header(struct header_filter_istream *mstream) { struct message_header_line *hdr; uoff_t highwater_offset; + size_t max_buffer_size; ssize_t ret, ret2; - bool matched; int hdr_ret; if (mstream->hdr_ctx == NULL) { @@ -181,19 +187,26 @@ static ssize_t read_header(struct header_filter_istream *mstream) } } + max_buffer_size = i_stream_get_max_buffer_size(&mstream->istream.istream); + if (mstream->hdr_buf->used >= max_buffer_size) + return -2; + while ((hdr_ret = message_parse_header_next(mstream->hdr_ctx, &hdr)) > 0) { - mstream->cur_line++; + bool matched; + if (!hdr->continued) + mstream->cur_line++; if (hdr->eoh) { mstream->seen_eoh = TRUE; matched = TRUE; - if (mstream->header_parsed) { + if (mstream->header_parsed && !mstream->headers_edited) { if (mstream->eoh_not_matched) matched = !matched; } else if (mstream->callback != NULL) { mstream->callback(mstream, hdr, &matched, mstream->context); + mstream->callbacks_called = TRUE; } if (!matched) { @@ -206,15 +219,25 @@ static ssize_t read_header(struct header_filter_istream *mstream) continue; } - matched = mstream->headers_count == 0 ? FALSE : - i_bsearch(hdr->name, mstream->headers, - mstream->headers_count, - sizeof(*mstream->headers), - bsearch_strcasecmp) != NULL; + if (hdr->continued) { + /* Header line continued - use only the first line's + matched-result. Otherwise multiline headers might + end up being only partially picked, which wouldn't + be very good. However, allow callbacks to modify + the headers in any way they want. */ + matched = mstream->prev_matched; + } else if (mstream->headers_count == 0) { + /* no include/exclude headers - default matching */ + matched = FALSE; + } else { + matched = i_bsearch(hdr->name, mstream->headers, + mstream->headers_count, + sizeof(*mstream->headers), + bsearch_strcasecmp) != NULL; + } if (mstream->callback == NULL) { /* nothing gets excluded */ - } else if (mstream->cur_line > mstream->parsed_lines || - mstream->headers_edited) { + } else if (!mstream->header_parsed || mstream->headers_edited) { /* first time in this line or we have actually modified the header so we always want to call the callbacks */ bool orig_matched = matched; @@ -222,19 +245,21 @@ static ssize_t read_header(struct header_filter_istream *mstream) mstream->parsed_lines = mstream->cur_line; mstream->callback(mstream, hdr, &matched, mstream->context); + mstream->callbacks_called = TRUE; if (matched != orig_matched && - !mstream->headers_edited) { + !hdr->continued && !mstream->headers_edited) { if (!array_is_created(&mstream->match_change_lines)) i_array_init(&mstream->match_change_lines, 8); array_append(&mstream->match_change_lines, &mstream->cur_line, 1); } - } else { + } else if (!hdr->continued) { /* second time in this line. was it excluded by the callback the first time? */ if (match_line_changed(mstream)) matched = !matched; } + mstream->prev_matched = matched; if (matched == mstream->exclude) { /* ignore */ @@ -263,6 +288,13 @@ static ssize_t read_header(struct header_filter_istream *mstream) break; } } + if (mstream->hdr_buf->used >= max_buffer_size) + break; + } + if (mstream->hdr_buf->used > 0) { + const unsigned char *data = mstream->hdr_buf->data; + mstream->last_added_newline = + data[mstream->hdr_buf->used-1] == '\n'; } if (hdr_ret < 0) { @@ -275,7 +307,9 @@ static ssize_t read_header(struct header_filter_istream *mstream) } if (!mstream->seen_eoh && mstream->add_missing_eoh) { mstream->seen_eoh = TRUE; - add_eol(mstream, FALSE); + if (!mstream->last_added_newline) + add_eol(mstream, mstream->last_orig_crlf); + add_eol(mstream, mstream->last_orig_crlf); } } @@ -294,11 +328,14 @@ static ssize_t read_header(struct header_filter_istream *mstream) message_parse_header_deinit(&mstream->hdr_ctx); mstream->hdr_ctx = NULL; - if (!mstream->header_parsed && mstream->callback != NULL) { + if ((!mstream->header_parsed || mstream->headers_edited || + mstream->callbacks_called) && + mstream->callback != NULL) { + bool matched = FALSE; mstream->callback(mstream, NULL, &matched, mstream->context); /* check if the callback added more headers. - this is allowed only of EOH wasn't added yet. */ + this is allowed only if EOH wasn't added yet. */ ret2 = hdr_stream_update_pos(mstream); if (!mstream->seen_eoh) ret += ret2; @@ -308,6 +345,7 @@ static ssize_t read_header(struct header_filter_istream *mstream) } mstream->header_parsed = TRUE; mstream->header_read = TRUE; + mstream->callbacks_called = FALSE; mstream->header_size.physical_size = mstream->istream.parent->v_offset; @@ -419,16 +457,17 @@ i_stream_header_filter_seek_to_header(struct header_filter_istream *mstream, message_parse_header_deinit(&mstream->hdr_ctx); mstream->skip_count = v_offset; mstream->cur_line = 0; + mstream->prev_matched = FALSE; mstream->header_read = FALSE; mstream->seen_eoh = FALSE; } -static void skip_header(struct header_filter_istream *mstream) +static int skip_header(struct header_filter_istream *mstream) { size_t pos; if (mstream->header_read) - return; + return 0; if (mstream->istream.access_counter != mstream->istream.parent->real_stream->access_counter) { @@ -441,6 +480,7 @@ static void skip_header(struct header_filter_istream *mstream) pos = i_stream_get_data_size(&mstream->istream.istream); i_stream_skip(&mstream->istream.istream, pos); } + return mstream->istream.istream.stream_errno != 0 ? -1 : 0; } static void @@ -479,7 +519,8 @@ static void i_stream_header_filter_seek(struct istream_private *stream, /* if we haven't parsed the whole header yet, we don't know if we want to seek inside header or body. so make sure we've parsed the header. */ - skip_header(mstream); + if (skip_header(mstream) < 0) + return; stream_reset_to(mstream, v_offset); if (v_offset < mstream->header_size.virtual_size) { @@ -519,7 +560,33 @@ i_stream_header_filter_stat(struct istream_private *stream, bool exact) /* fix the filtered header size */ old_offset = stream->istream.v_offset; - skip_header(mstream); + if (skip_header(mstream) < 0) + return -1; + + if (mstream->hide_body) { + /* no body */ + stream->statbuf.st_size = mstream->header_size.physical_size; + } else if (!mstream->end_body_with_lf) { + /* no last-LF */ + } else if (mstream->last_lf_added) { + /* yes, we have added LF */ + stream->statbuf.st_size += mstream->crlf ? 2 : 1; + } else if (mstream->last_lf_offset != (uoff_t)-1) { + /* no, we didn't need to add LF */ + } else { + /* check if we need to add LF */ + i_stream_seek(stream->parent, st->st_size - 1); + (void)i_stream_read(stream->parent); + if (stream->parent->stream_errno != 0) { + stream->istream.stream_errno = + stream->parent->stream_errno; + return -1; + } + i_assert(stream->parent->eof); + ssize_t ret = handle_end_body_with_lf(mstream, -1); + if (ret > 0) + stream->statbuf.st_size += ret; + } stream->statbuf.st_size -= (off_t)mstream->header_size.physical_size - @@ -575,6 +642,7 @@ i_stream_create_header_filter(struct istream *input, mstream->add_missing_eoh = (flags & HEADER_FILTER_ADD_MISSING_EOH) != 0; mstream->end_body_with_lf = (flags & HEADER_FILTER_END_BODY_WITH_LF) != 0; + mstream->last_lf_offset = (uoff_t)-1; mstream->istream.iostream.destroy = i_stream_header_filter_destroy; mstream->istream.read = i_stream_header_filter_read; diff --git a/src/lib-mail/istream-qp-decoder.c b/src/lib-mail/istream-qp-decoder.c index 3dfdd74dacd..611defe5cb0 100644 --- a/src/lib-mail/istream-qp-decoder.c +++ b/src/lib-mail/istream-qp-decoder.c @@ -32,24 +32,42 @@ static ssize_t i_stream_qp_decoder_read(struct istream_private *stream) struct qp_decoder_istream *bstream = (struct qp_decoder_istream *)stream; const unsigned char *data; - size_t size, avail, error_pos; + size_t size, error_pos, max_buffer_size; const char *error; int ret; + max_buffer_size = i_stream_get_max_buffer_size(&stream->istream); for (;;) { + /* remove skipped data from buffer */ + if (stream->skip > 0) { + i_assert(stream->skip <= bstream->buf->used); + buffer_delete(bstream->buf, 0, stream->skip); + stream->pos -= stream->skip; + stream->skip = 0; + } + + stream->buffer = bstream->buf->data; + + i_assert(stream->pos <= bstream->buf->used); + if (stream->pos >= max_buffer_size) { + /* stream buffer still at maximum */ + return -2; + } + /* if something is already decoded, return as much of it as we can */ if (bstream->buf->used > 0) { - i_stream_try_alloc(stream, bstream->buf->used, &avail); - if (avail == 0) - return -2; - size = I_MIN(avail, bstream->buf->used); - memcpy(stream->w_buffer + stream->pos, - bstream->buf->data, size); - buffer_delete(bstream->buf, 0, size); - stream->pos += size; - return size; + size_t new_pos, bytes; + + /* only return up to max_buffer_size bytes, even when buffer + actually has more, as not to confuse the caller */ + new_pos = I_MIN(bstream->buf->used, max_buffer_size); + bytes = new_pos - stream->pos; + stream->pos = new_pos; + + return (ssize_t)bytes; } + /* need to read more input */ ret = i_stream_read_data(stream->parent, &data, &size, 0); if (ret <= 0) { diff --git a/src/lib-mail/message-date.c b/src/lib-mail/message-date.c index 49fedd8a545..ab52e783277 100644 --- a/src/lib-mail/message-date.c +++ b/src/lib-mail/message-date.c @@ -222,7 +222,11 @@ message_date_parser_tokens(struct message_date_parser_context *ctx, /* missing timezone */ *timezone_offset_r = 0; } else { - /* timezone */ + /* timezone. invalid timezones are treated as GMT, because + we may not know all the possible timezones that are used + and it's better to give at least a mostly correct reply. + FIXME: perhaps some different strict version of this + function would be useful? */ *timezone_offset_r = parse_timezone(value, len); } diff --git a/src/lib-mail/message-header-parser.c b/src/lib-mail/message-header-parser.c index 449667d750f..587048fc589 100644 --- a/src/lib-mail/message-header-parser.c +++ b/src/lib-mail/message-header-parser.c @@ -33,6 +33,7 @@ message_parse_header_init(struct istream *input, struct message_size *hdr_size, ctx->name = str_new(default_pool, 128); ctx->flags = flags; ctx->value_buf = buffer_create_dynamic(default_pool, 4096); + i_stream_ref(input); if (hdr_size != NULL) memset(hdr_size, 0, sizeof(*hdr_size)); @@ -43,6 +44,7 @@ void message_parse_header_deinit(struct message_header_parser_ctx **_ctx) { struct message_header_parser_ctx *ctx = *_ctx; + i_stream_unref(&ctx->input); buffer_free(&ctx->value_buf); str_free(&ctx->name); i_free(ctx); diff --git a/src/lib-mail/message-parser.c b/src/lib-mail/message-parser.c index a8177f37857..20ad1027256 100644 --- a/src/lib-mail/message-parser.c +++ b/src/lib-mail/message-parser.c @@ -45,6 +45,7 @@ struct message_parser_ctx { struct message_block *block_r); unsigned int part_seen_content_type:1; + unsigned int multipart:1; unsigned int eof:1; }; @@ -65,19 +66,23 @@ static struct message_boundary * boundary_find(struct message_boundary *boundaries, const unsigned char *data, size_t len) { + struct message_boundary *best = NULL; + /* As MIME spec says: search from latest one to oldest one so that we don't break if the same boundary is used in nested parts. Also the full message line doesn't have to match the boundary, only the - beginning. */ + beginning. However, if there are multiple prefixes whose beginning + matches, use the longest matching one. */ while (boundaries != NULL) { if (boundaries->len <= len && - memcmp(boundaries->boundary, data, boundaries->len) == 0) - return boundaries; + memcmp(boundaries->boundary, data, boundaries->len) == 0 && + (best == NULL || best->len < boundaries->len)) + best = boundaries; boundaries = boundaries->next; } - return NULL; + return best; } static void parse_body_add_block(struct message_parser_ctx *ctx, @@ -168,6 +173,8 @@ message_part_append(pool_t pool, struct message_part *parent) struct message_part *p, *part, **list; i_assert(parent != NULL); + i_assert((parent->flags & (MESSAGE_PART_FLAG_MULTIPART | + MESSAGE_PART_FLAG_MESSAGE_RFC822)) != 0); part = p_new(pool, struct message_part, 1); part->parent = parent; @@ -308,6 +315,8 @@ static int parse_part_finish(struct message_parser_ctx *ctx, struct message_part *part; size_t line_size; + i_assert(ctx->last_boundary == NULL); + /* get back to parent MIME part, summing the child MIME part sizes into parent's body sizes */ for (part = ctx->part; part != boundary->part; part = part->parent) { @@ -504,6 +513,21 @@ static void parse_content_type(struct message_parser_ctx *ctx, } } +static bool block_is_at_eoh(const struct message_block *block) +{ + if (block->size < 1) + return FALSE; + if (block->data[0] == '\n') + return TRUE; + if (block->data[0] == '\r') { + if (block->size < 2) + return FALSE; + if (block->data[1] == '\n') + return TRUE; + } + return FALSE; +} + #define MUTEX_FLAGS \ (MESSAGE_PART_FLAG_MESSAGE_RFC822 | MESSAGE_PART_FLAG_MULTIPART) @@ -519,12 +543,30 @@ static int parse_next_header(struct message_parser_ctx *ctx, if ((ret = message_parser_read_more(ctx, block_r, &full)) == 0) return ret; + if (ret > 0 && block_is_at_eoh(block_r) && + ctx->last_boundary != NULL && + (part->flags & MESSAGE_PART_FLAG_IS_MIME) != 0) { + /* we are at the end of headers and we've determined that we're + going to start a multipart. add the boundary already here + at this point so we can reliably determine whether the + "\n--boundary" belongs to us or to a previous boundary. + this is a problem if the boundary prefixes are identical, + because MIME requires only the prefix to match. */ + parse_next_body_multipart_init(ctx); + ctx->multipart = TRUE; + } + /* before parsing the header see if we can find a --boundary from here. we're guaranteed to be at the beginning of the line here. */ if (ret > 0) { ret = ctx->boundaries == NULL ? -1 : boundary_line_find(ctx, block_r->data, block_r->size, full, &boundary); + if (ret > 0 && boundary->part == ctx->part) { + /* our own body begins with our own --boundary. + we don't want to handle that yet. */ + ret = -1; + } } if (ret < 0) { /* no boundary */ @@ -581,17 +623,13 @@ static int parse_next_header(struct message_parser_ctx *ctx, } /* end of headers */ - if ((part->flags & MESSAGE_PART_FLAG_MULTIPART) != 0 && - ctx->last_boundary == NULL) { - /* multipart type but no message boundary */ - part->flags = 0; - } if ((part->flags & MESSAGE_PART_FLAG_IS_MIME) == 0) { /* It's not MIME. Reset everything we found from Content-Type. */ + i_assert(!ctx->multipart); part->flags = 0; - ctx->last_boundary = NULL; } + ctx->last_boundary = NULL; if (!ctx->part_seen_content_type || (part->flags & MESSAGE_PART_FLAG_IS_MIME) == 0) { @@ -615,8 +653,9 @@ static int parse_next_header(struct message_parser_ctx *ctx, i_assert((part->flags & MUTEX_FLAGS) != MUTEX_FLAGS); ctx->last_chr = '\n'; - if (ctx->last_boundary != NULL) { - parse_next_body_multipart_init(ctx); + if (ctx->multipart) { + i_assert(ctx->last_boundary == NULL); + ctx->multipart = FALSE; ctx->parse_next_block = parse_next_body_to_boundary; } else if (part->flags & MESSAGE_PART_FLAG_MESSAGE_RFC822) ctx->parse_next_block = parse_next_body_message_rfc822_init; @@ -966,14 +1005,22 @@ static int preparsed_parse_next_header(struct message_parser_ctx *ctx, static int preparsed_parse_next_header_init(struct message_parser_ctx *ctx, struct message_block *block_r) { + struct istream *hdr_input; + i_assert(ctx->hdr_parser_ctx == NULL); i_assert(ctx->part->physical_pos >= ctx->input->v_offset); i_stream_skip(ctx->input, ctx->part->physical_pos - ctx->input->v_offset); + /* the header may become truncated by --boundaries. limit the header + stream's size to what it's supposed to be to avoid duplicating (and + keeping in sync!) all the same complicated logic as in + parse_next_header(). */ + hdr_input = i_stream_create_limit(ctx->input, ctx->part->header_size.physical_size); ctx->hdr_parser_ctx = - message_parse_header_init(ctx->input, NULL, ctx->hdr_flags); + message_parse_header_init(hdr_input, NULL, ctx->hdr_flags); + i_stream_unref(&hdr_input); ctx->parse_next_block = preparsed_parse_next_header; return preparsed_parse_next_header(ctx, block_r); diff --git a/src/lib-mail/test-istream-header-filter.c b/src/lib-mail/test-istream-header-filter.c index 6d6629116ac..46c90ca93fb 100644 --- a/src/lib-mail/test-istream-header-filter.c +++ b/src/lib-mail/test-istream-header-filter.c @@ -7,14 +7,39 @@ #include "istream-header-filter.h" #include "test-common.h" +struct run_ctx { + header_filter_callback *callback; + bool null_hdr_seen; + bool callback_called; +}; + +static void run_callback(struct header_filter_istream *input, + struct message_header_line *hdr, + bool *matched, struct run_ctx *ctx) +{ + if (hdr == NULL) + ctx->null_hdr_seen = TRUE; + if (ctx->callback != NULL) + ctx->callback(input, hdr, matched, NULL); + ctx->callback_called = TRUE; +} + static void -test_istream_run(struct istream *test_istream, struct istream *filter, - unsigned int input_len, const char *output) +test_istream_run(struct istream *test_istream, + unsigned int input_len, const char *output, + enum header_filter_flags flags, + header_filter_callback *callback) { + struct run_ctx run_ctx = { .callback = callback, .null_hdr_seen = FALSE }; + struct istream *filter; unsigned int i, output_len = strlen(output); + const struct stat *st; const unsigned char *data; size_t size; + filter = i_stream_create_header_filter(test_istream, flags, NULL, 0, + run_callback, &run_ctx); + for (i = 1; i < input_len; i++) { test_istream_set_size(test_istream, i); test_assert(i_stream_read(filter) >= 0); @@ -23,6 +48,10 @@ test_istream_run(struct istream *test_istream, struct istream *filter, test_assert(i_stream_read(filter) > 0); test_assert(i_stream_read(filter) == -1); + test_assert(run_ctx.null_hdr_seen); + run_ctx.null_hdr_seen = FALSE; + run_ctx.callback_called = FALSE; + data = i_stream_get_data(filter, &size); test_assert(size == output_len && memcmp(data, output, size) == 0); @@ -30,8 +59,12 @@ test_istream_run(struct istream *test_istream, struct istream *filter, i_stream_skip(filter, size); i_stream_seek(filter, 0); while (i_stream_read(filter) > 0) ; + test_assert(run_ctx.null_hdr_seen == run_ctx.callback_called); data = i_stream_get_data(filter, &size); test_assert(size == output_len && memcmp(data, output, size) == 0); + test_assert(i_stream_stat(filter, TRUE, &st) == 0 && + (uoff_t)st->st_size == size); + i_stream_unref(&filter); } static void ATTR_NULL(3) @@ -48,26 +81,29 @@ filter_callback(struct header_filter_istream *input ATTR_UNUSED, static void test_istream_filter(void) { - static const char *exclude_headers[] = { "Subject", "To", "X-Drop", NULL }; + static const char *exclude_headers[] = { "Subject", "To" }; const char *input = "From: foo\nFrom: abc\nTo: bar\nSubject: plop\nX-Drop: 1\n\nhello world\n"; const char *output = "From: abc\n\nhello world\n"; struct istream *istream, *filter, *filter2; unsigned int i, input_len = strlen(input); unsigned int output_len = strlen(output); const unsigned char *data; + const struct stat *st; size_t size; - test_begin("i_stream_create_header_filter(exclude)"); + test_begin("i_stream_create_header_filter: exclude"); istream = test_istream_create(input); filter = i_stream_create_header_filter(istream, HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR, - exclude_headers, 2, + exclude_headers, + N_ELEMENTS(exclude_headers), filter_callback, (void *)NULL); filter2 = i_stream_create_header_filter(filter, HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR, - exclude_headers, 2, + exclude_headers, + N_ELEMENTS(exclude_headers), *null_header_filter_callback, (void *)NULL); i_stream_unref(&filter); @@ -84,12 +120,225 @@ static void test_istream_filter(void) data = i_stream_get_data(filter, &size); test_assert(size == output_len && memcmp(data, output, size) == 0); + test_assert(i_stream_stat(filter, TRUE, &st) == 0 && + (uoff_t)st->st_size == size); + i_stream_skip(filter, size); i_stream_seek(filter, 0); while (i_stream_read(filter) > 0) ; data = i_stream_get_data(filter, &size); test_assert(size == output_len && memcmp(data, output, size) == 0); + test_assert(i_stream_stat(filter, TRUE, &st) == 0 && + (uoff_t)st->st_size == size); + + i_stream_unref(&filter); + i_stream_unref(&istream); + + test_end(); +} + +static void add_random_text(string_t *dest, unsigned int count) +{ + unsigned int i; + + for (i = 0; i < count; i++) + str_append_c(dest, rand() % ('z'-'a'+1) + 'a'); +} + +static void ATTR_NULL(3) +filter2_callback(struct header_filter_istream *input ATTR_UNUSED, + struct message_header_line *hdr, + bool *matched, bool *null_hdr_seen) +{ + if (hdr == NULL) + *null_hdr_seen = TRUE; + else if (strcmp(hdr->name, "To") == 0) + *matched = TRUE; +} + +static void test_istream_filter_large_buffer(void) +{ + string_t *input, *output; + struct istream *istream, *filter; + const struct stat *st; + const unsigned char *data; + size_t size, prefix_len; + const char *p; + unsigned int i; + bool null_hdr_seen = FALSE; + + test_begin("i_stream_create_header_filter: large buffer"); + + input = str_new(default_pool, 1024*128); + output = str_new(default_pool, 1024*128); + str_append(input, "From: "); + add_random_text(input, 1024*31); + str_append(input, "\nTo: "); + add_random_text(input, 1024*32); + str_append(input, "\nSubject: "); + add_random_text(input, 1024*34); + str_append(input, "\n\nbody\n"); + + istream = test_istream_create_data(str_data(input), str_len(input)); + test_istream_set_max_buffer_size(istream, 8192); + + filter = i_stream_create_header_filter(istream, + HEADER_FILTER_EXCLUDE | + HEADER_FILTER_NO_CR, + NULL, 0, + filter2_callback, + &null_hdr_seen); + + for (i = 0; i < 2; i++) { + for (;;) { + ssize_t ret = i_stream_read(filter); + i_assert(ret != 0); + if (ret == -1) + break; + if (ret == -2) { + data = i_stream_get_data(filter, &size); + str_append_n(output, data, size); + i_stream_skip(filter, size); + } + } + /* callbacks are called only once */ + test_assert(null_hdr_seen == (i == 0)); + + data = i_stream_get_data(filter, &size); + test_assert(size <= 8192); + str_append_n(output, data, size); + + p = strstr(str_c(input), "To: "); + i_assert(p != NULL); + prefix_len = p - str_c(input); + test_assert(strncmp(str_c(input), str_c(output), prefix_len) == 0); + + p = strchr(p, '\n'); + i_assert(p != NULL); + test_assert(strcmp(p+1, str_c(output) + prefix_len) == 0); + + test_assert(i_stream_stat(filter, TRUE, &st) == 0 && + (uoff_t)st->st_size == filter->v_offset + size); + + /* seek back and retry once with caching and different + buffer size */ + i_stream_seek(filter, 0); + str_truncate(output, 0); + test_istream_set_max_buffer_size(istream, 4096); + null_hdr_seen = FALSE; + } + + str_free(&input); + str_free(&output); + i_stream_unref(&filter); + i_stream_unref(&istream); + + test_end(); +} + +static void test_istream_filter_large_buffer2(void) +{ + static const char *wanted_headers[] = { "References" }; + string_t *input, *output; + struct istream *istream, *filter; + const struct stat *st; + const unsigned char *data; + size_t size; + unsigned int i; + int ret; + + test_begin("i_stream_create_header_filter: large buffer2"); + + input = str_new(default_pool, 1024*128); + output = str_new(default_pool, 1024*128); + str_append(input, "References: "); + add_random_text(input, 1024*64); + str_append(input, "\r\n\r\n"); + + istream = test_istream_create_data(str_data(input), str_len(input)); + test_istream_set_max_buffer_size(istream, 8192); + + filter = i_stream_create_header_filter(istream, + HEADER_FILTER_INCLUDE | HEADER_FILTER_HIDE_BODY, + wanted_headers, N_ELEMENTS(wanted_headers), + *null_header_filter_callback, (void *)NULL); + + for (i = 0; i < 2; i++) { + while ((ret = i_stream_read_more(filter, &data, &size)) > 0) { + str_append_n(output, data, size); + i_stream_skip(filter, size); + } + test_assert(ret == -1); + test_assert(filter->stream_errno == 0); + + test_assert(strcmp(str_c(input), str_c(output)) == 0); + test_assert(i_stream_stat(filter, TRUE, &st) == 0 && + (uoff_t)st->st_size == filter->v_offset + size); + + /* seek back and retry once with caching and different + buffer size */ + i_stream_seek(filter, 0); + str_truncate(output, 0); + test_istream_set_max_buffer_size(istream, 4096); + } + + str_free(&input); + str_free(&output); + i_stream_unref(&filter); + i_stream_unref(&istream); + test_end(); +} + +static void +filter3_callback(struct header_filter_istream *input ATTR_UNUSED, + struct message_header_line *hdr, + bool *matched ATTR_UNUSED, string_t *dest) +{ + if (hdr != NULL) + message_header_line_write(dest, hdr); +} + +static void test_istream_callbacks(void) +{ + string_t *input, *output; + const struct stat *st; + struct istream *istream, *filter; + unsigned int i; + + test_begin("i_stream_create_header_filter: callbacks"); + + input = str_new(default_pool, 1024*128); + output = str_new(default_pool, 1024*128); + str_append(input, "From: first line\n "); + add_random_text(input, 1024*31); + str_append(input, "\nTo: first line\n\tsecond line\n\t"); + add_random_text(input, 1024*32); + str_append(input, "\n last line\nSubject: "); + add_random_text(input, 1024*34); + str_append(input, "\n"); + + istream = test_istream_create_data(str_data(input), str_len(input)); + test_istream_set_max_buffer_size(istream, 8192); + + filter = i_stream_create_header_filter(istream, + HEADER_FILTER_EXCLUDE | + HEADER_FILTER_NO_CR, + NULL, 0, + filter3_callback, + output); + + /* callback should be called exactly once for all the header input */ + for (i = 0; i < 2; i++) { + while (i_stream_read(filter) != -1) + i_stream_skip(filter, i_stream_get_data_size(filter)); + } + + test_assert(i_stream_stat(filter, TRUE, &st) == 0 && + (uoff_t)st->st_size == str_len(output)); + test_assert(strcmp(str_c(output), str_c(input)) == 0); + str_free(&input); + str_free(&output); i_stream_unref(&filter); i_stream_unref(&istream); @@ -101,7 +350,14 @@ edit_callback(struct header_filter_istream *input, struct message_header_line *hdr, bool *matched, void *context ATTR_UNUSED) { - if (hdr != NULL && strcasecmp(hdr->name, "To") == 0) { + if (hdr == NULL) + return; + if (hdr->eoh) { + /* add a new header */ + const char *new_hdr = "Added: header\n\n"; + i_stream_header_filter_add(input, new_hdr, strlen(new_hdr)); + *matched = FALSE; + } else if (strcasecmp(hdr->name, "To") == 0) { /* modify To header */ const char *new_to = "To: 123\n"; *matched = TRUE; @@ -112,18 +368,15 @@ edit_callback(struct header_filter_istream *input, static void test_istream_edit(void) { const char *input = "From: foo\nTo: bar\n\nhello world\n"; - const char *output = "From: foo\nTo: 123\n\nhello world\n"; - struct istream *istream, *filter; + const char *output = "From: foo\nTo: 123\nAdded: header\n\nhello world\n"; + struct istream *istream; - test_begin("i_stream_create_header_filter(edit)"); + test_begin("i_stream_create_header_filter: edit headers"); istream = test_istream_create(input); - filter = i_stream_create_header_filter(istream, - HEADER_FILTER_EXCLUDE | - HEADER_FILTER_NO_CR, - NULL, 0, - edit_callback, (void *)NULL); - test_istream_run(istream, filter, strlen(input), output); - i_stream_unref(&filter); + test_istream_run(istream, strlen(input), output, + HEADER_FILTER_EXCLUDE | + HEADER_FILTER_NO_CR, + edit_callback); i_stream_unref(&istream); test_end(); @@ -131,9 +384,9 @@ static void test_istream_edit(void) static void test_istream_end_body_with_lf(void) { - static const char *empty_strarray[] = { NULL }; const char *input = "From: foo\n\nhello world"; const char *output = "From: foo\n\nhello world\n"; + const struct stat *st; struct istream *istream, *filter; unsigned int i, input_len = strlen(input); unsigned int output_len = strlen(output); @@ -141,13 +394,13 @@ static void test_istream_end_body_with_lf(void) string_t *str = t_str_new(64); size_t size; - test_begin("i_stream_create_header_filter(end_body_with_lf)"); + test_begin("i_stream_create_header_filter: end_body_with_lf"); istream = test_istream_create(input); filter = i_stream_create_header_filter(istream, HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR | HEADER_FILTER_END_BODY_WITH_LF, - empty_strarray, 0, + NULL, 0, *null_header_filter_callback, (void *)NULL); @@ -162,6 +415,8 @@ static void test_istream_end_body_with_lf(void) data = i_stream_get_data(filter, &size); test_assert(size == output_len && memcmp(data, output, size) == 0); + test_assert(i_stream_stat(filter, TRUE, &st) == 0 && + (uoff_t)st->st_size == filter->v_offset + size); i_stream_skip(filter, size); i_stream_seek(filter, 0); @@ -188,6 +443,72 @@ static void test_istream_end_body_with_lf(void) test_end(); } +static void test_istream_add_missing_eoh(void) +{ + struct { + const char *input; + const char *output; + unsigned int extra; + } tests[] = { + { "From: foo", "From: foo\n\n", 1 }, + { "From: foo\n", "From: foo\n\n", 1 }, + { "From: foo\n\n", "From: foo\n\n", 1 }, + { "From: foo\n\nbar", "From: foo\n\nbar", 0 }, + { "From: foo\r\n", "From: foo\r\n\r\n", 1 }, + { "From: foo\r\n\r\n", "From: foo\r\n\r\n", 0 }, + { "From: foo\r\n\r\nbar", "From: foo\r\n\r\nbar", 0 } + }; + struct istream *istream; + unsigned int i; + + test_begin("i_stream_create_header_filter: add missing EOH"); + for (i = 0; i < N_ELEMENTS(tests); i++) { + istream = test_istream_create(tests[i].input); + test_istream_run(istream, + strlen(tests[i].input) + tests[i].extra, + tests[i].output, + HEADER_FILTER_EXCLUDE | + HEADER_FILTER_CRLF_PRESERVE | + HEADER_FILTER_ADD_MISSING_EOH, + *null_header_filter_callback); + i_stream_unref(&istream); + } + test_end(); +} + +static void test_istream_hide_body(void) +{ + struct { + const char *input; + const char *output; + int extra; + } tests[] = { + { "From: foo", "From: foo", 0 }, + { "From: foo\n", "From: foo\n", 0 }, + { "From: foo\n\n", "From: foo\n\n", 1 }, + { "From: foo\n\nbar", "From: foo\n\n", -2 }, + { "From: foo\r\n", "From: foo\r\n", 0 }, + { "From: foo\r\n\r\n", "From: foo\r\n\r\n", 0 }, + { "From: foo\r\n\r\nbar", "From: foo\r\n\r\n", -3 } + }; + struct istream *istream; + unsigned int i; + + test_begin("i_stream_create_header_filter: hide body"); + for (i = 0; i < N_ELEMENTS(tests); i++) { + istream = test_istream_create(tests[i].input); + test_istream_run(istream, + strlen(tests[i].input) + tests[i].extra, + tests[i].output, + HEADER_FILTER_EXCLUDE | + HEADER_FILTER_CRLF_PRESERVE | + HEADER_FILTER_HIDE_BODY, + *null_header_filter_callback); + i_stream_unref(&istream); + } + test_end(); +} + static void ATTR_NULL(3) strip_eoh_callback(struct header_filter_istream *input ATTR_UNUSED, struct message_header_line *hdr, @@ -201,17 +522,56 @@ static void test_istream_strip_eoh(void) { const char *input = "From: foo\nTo: bar\n\nhello world\n"; const char *output = "From: foo\nTo: bar\nhello world\n"; - struct istream *istream, *filter; + struct istream *istream; - test_begin("i_stream_create_header_filter(edit)"); + test_begin("i_stream_create_header_filter: strip_eoh"); istream = test_istream_create(input); - filter = i_stream_create_header_filter(istream, - HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR, NULL, 0, - strip_eoh_callback, (void *)NULL); - test_istream_run(istream, filter, strlen(input), output); - i_stream_unref(&filter); + test_istream_run(istream, strlen(input), output, + HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR, + strip_eoh_callback); + i_stream_unref(&istream); + + test_end(); +} + +static void ATTR_NULL(3) +missing_eoh_callback(struct header_filter_istream *input ATTR_UNUSED, + struct message_header_line *hdr, + bool *matched ATTR_UNUSED, void *context ATTR_UNUSED) +{ + if (hdr == NULL) { + const char *new_hdr = "Subject: added\n\n"; + i_stream_header_filter_add(input, new_hdr, strlen(new_hdr)); + } +} + +static void test_istream_missing_eoh_callback(void) +{ + const char *input = "From: foo\nTo: bar\n"; + const char *output = "From: foo\nTo: bar\nSubject: added\n\n"; + struct istream *istream; + + test_begin("i_stream_create_header_filter: add headers when EOH is missing"); + istream = test_istream_create(input); + test_istream_run(istream, strlen(input) + 1, output, + HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR, + missing_eoh_callback); i_stream_unref(&istream); + test_end(); +} +static void test_istream_empty_missing_eoh_callback(void) +{ + const char *input = ""; + const char *output = "Subject: added\n\n"; + struct istream *istream; + + test_begin("i_stream_create_header_filter: add headers when mail is empty"); + istream = test_istream_create(input); + test_istream_run(istream, strlen(input)+1, output, + HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR, + missing_eoh_callback); + i_stream_unref(&istream); test_end(); } @@ -219,9 +579,16 @@ int main(void) { static void (*test_functions[])(void) = { test_istream_filter, + test_istream_filter_large_buffer, + test_istream_filter_large_buffer2, + test_istream_callbacks, test_istream_edit, + test_istream_add_missing_eoh, test_istream_end_body_with_lf, + test_istream_hide_body, test_istream_strip_eoh, + test_istream_missing_eoh_callback, + test_istream_empty_missing_eoh_callback, NULL }; return test_run(test_functions); diff --git a/src/lib-mail/test-message-parser.c b/src/lib-mail/test-message-parser.c index da56fb05686..66360d49053 100644 --- a/src/lib-mail/test-message-parser.c +++ b/src/lib-mail/test-message-parser.c @@ -68,6 +68,32 @@ static bool msg_parts_cmp(struct message_part *p1, struct message_part *p2) return TRUE; } +static void test_parsed_parts(struct istream *input, struct message_part *parts) +{ + struct message_parser_ctx *parser; + struct message_block block; + struct message_part *parts2; + uoff_t i, input_size; + const char *error; + int ret; + + i_stream_seek(input, 0); + if (i_stream_get_size(input, TRUE, &input_size) < 0) + i_unreached(); + + parser = message_parser_init_from_parts(parts, input, 0, + MESSAGE_PARSER_FLAG_SKIP_BODY_BLOCK); + for (i = 1; i <= input_size*2+1; i++) { + test_istream_set_size(input, i/2); + if (i > TEST_MSG_LEN*2) + test_istream_set_allow_eof(input, TRUE); + while ((ret = message_parser_parse_next_block(parser, + &block)) > 0) ; + } + test_assert(message_parser_deinit_from_parts(&parser, &parts2, &error) == 0); + test_assert(msg_parts_cmp(parts, parts2)); +} + static void test_message_parser_small_blocks(void) { struct message_parser_ctx *parser; @@ -170,18 +196,25 @@ static const char input_msg[] = test_assert(message_parser_deinit(&parser, &parts) == 0); test_assert((parts->flags & MESSAGE_PART_FLAG_MULTIPART) != 0); + test_assert(parts->header_size.lines == 2); + test_assert(parts->header_size.physical_size == 48); + test_assert(parts->header_size.virtual_size == 48+2); test_assert(parts->body_size.lines == 8); test_assert(parts->body_size.physical_size == 112); test_assert(parts->body_size.virtual_size == 112+7); + test_assert(parts->children->physical_pos == 55); test_assert(parts->children->header_size.physical_size == 0); test_assert(parts->children->body_size.physical_size == 0); test_assert(parts->children->body_size.lines == 0); + test_assert(parts->children->next->physical_pos == 62); test_assert(parts->children->next->header_size.physical_size == 24); test_assert(parts->children->next->header_size.virtual_size == 24); test_assert(parts->children->next->header_size.lines == 0); + test_assert(parts->children->next->next->physical_pos == 94); test_assert(parts->children->next->next->header_size.physical_size == 24); test_assert(parts->children->next->next->header_size.virtual_size == 24); test_assert(parts->children->next->next->header_size.lines == 0); + test_assert(parts->children->next->next->next->physical_pos == 127); test_assert(parts->children->next->next->next->header_size.physical_size == 23); test_assert(parts->children->next->next->next->header_size.virtual_size == 23); test_assert(parts->children->next->next->next->header_size.lines == 0); @@ -191,6 +224,391 @@ static const char input_msg[] = } test_assert(parts->children->next->next->next->next == NULL); + test_parsed_parts(input, parts); + i_stream_unref(&input); + pool_unref(&pool); + test_end(); +} + +static void test_message_parser_truncated_mime_headers2(void) +{ +static const char input_msg[] = +"Content-Type: multipart/mixed; boundary=\"ab\"\n" +"\n" +"--ab\n" +"Content-Type: multipart/mixed; boundary=\"a\"\n" +"\n" +"--ab\n" +"Content-Type: text/plain\n" +"\n" +"--a\n\n"; + struct message_parser_ctx *parser; + struct istream *input; + struct message_part *parts; + struct message_block block; + pool_t pool; + int ret; + + test_begin("message parser truncated mime headers 2"); + pool = pool_alloconly_create("message parser", 10240); + input = test_istream_create(input_msg); + + parser = message_parser_init(pool, input, 0, 0); + while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ; + test_assert(ret < 0); + test_assert(message_parser_deinit(&parser, &parts) == 0); + + test_assert(parts->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(parts->header_size.lines == 2); + test_assert(parts->header_size.physical_size == 46); + test_assert(parts->header_size.virtual_size == 46+2); + test_assert(parts->body_size.lines == 8); + test_assert(parts->body_size.physical_size == 86); + test_assert(parts->body_size.virtual_size == 86+8); + + test_assert(parts->children->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(parts->children->physical_pos == 51); + test_assert(parts->children->header_size.lines == 1); + test_assert(parts->children->header_size.physical_size == 44); + test_assert(parts->children->header_size.virtual_size == 44+1); + test_assert(parts->children->body_size.lines == 0); + test_assert(parts->children->body_size.physical_size == 0); + test_assert(parts->children->children == NULL); + + test_assert(parts->children->next->flags == (MESSAGE_PART_FLAG_TEXT | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(parts->children->next->physical_pos == 101); + test_assert(parts->children->next->header_size.lines == 2); + test_assert(parts->children->next->header_size.physical_size == 26); + test_assert(parts->children->next->header_size.virtual_size == 26+2); + test_assert(parts->children->next->body_size.lines == 2); + test_assert(parts->children->next->body_size.physical_size == 5); + test_assert(parts->children->next->body_size.virtual_size == 5+2); + test_assert(parts->children->next->children == NULL); + + test_parsed_parts(input, parts); + i_stream_unref(&input); + pool_unref(&pool); + test_end(); +} + +static void test_message_parser_truncated_mime_headers3(void) +{ +static const char input_msg[] = +"Content-Type: multipart/mixed; boundary=\"ab\"\n"; + struct message_parser_ctx *parser; + struct istream *input; + struct message_part *parts; + struct message_block block; + pool_t pool; + int ret; + + test_begin("message parser truncated mime headers 3"); + pool = pool_alloconly_create("message parser", 10240); + input = test_istream_create(input_msg); + + parser = message_parser_init(pool, input, 0, 0); + while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ; + test_assert(ret < 0); + test_assert(message_parser_deinit(&parser, &parts) == 0); + + test_assert(parts->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(parts->header_size.lines == 1); + test_assert(parts->header_size.physical_size == 45); + test_assert(parts->header_size.virtual_size == 45+1); + test_assert(parts->body_size.lines == 0); + test_assert(parts->body_size.physical_size == 0); + + test_assert(parts->children == NULL); + + test_parsed_parts(input, parts); + i_stream_unref(&input); + pool_unref(&pool); + test_end(); +} + +static void test_message_parser_empty_multipart(void) +{ +static const char input_msg[] = +"Content-Type: multipart/mixed; boundary=\"ab\"\n" +"\n" +"body\n"; + struct message_parser_ctx *parser; + struct istream *input; + struct message_part *parts; + struct message_block block; + pool_t pool; + int ret; + + test_begin("message parser empty multipart"); + pool = pool_alloconly_create("message parser", 10240); + input = test_istream_create(input_msg); + + parser = message_parser_init(pool, input, 0, 0); + while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ; + test_assert(ret < 0); + test_assert(message_parser_deinit(&parser, &parts) == 0); + + test_assert(parts->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(parts->header_size.lines == 2); + test_assert(parts->header_size.physical_size == 46); + test_assert(parts->header_size.virtual_size == 46+2); + test_assert(parts->body_size.lines == 1); + test_assert(parts->body_size.physical_size == 5); + test_assert(parts->body_size.virtual_size == 5+1); + + test_assert(parts->children == NULL); + + test_parsed_parts(input, parts); + i_stream_unref(&input); + pool_unref(&pool); + test_end(); +} + +static void test_message_parser_duplicate_mime_boundary(void) +{ +static const char input_msg[] = +"Content-Type: multipart/mixed; boundary=\"a\"\n" +"\n" +"--a\n" +"Content-Type: multipart/mixed; boundary=\"a\"\n" +"\n" +"--a\n" +"Content-Type: text/plain\n" +"\n" +"body\n"; + struct message_parser_ctx *parser; + struct istream *input; + struct message_part *parts; + struct message_block block; + pool_t pool; + int ret; + + test_begin("message parser duplicate mime boundary"); + pool = pool_alloconly_create("message parser", 10240); + input = test_istream_create(input_msg); + + parser = message_parser_init(pool, input, 0, 0); + while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ; + test_assert(ret < 0); + test_assert(message_parser_deinit(&parser, &parts) == 0); + + test_assert(parts->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(parts->header_size.lines == 2); + test_assert(parts->header_size.physical_size == 45); + test_assert(parts->header_size.virtual_size == 45+2); + test_assert(parts->body_size.lines == 7); + test_assert(parts->body_size.physical_size == 84); + test_assert(parts->body_size.virtual_size == 84+7); + test_assert(parts->children->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(parts->children->physical_pos == 49); + test_assert(parts->children->header_size.lines == 2); + test_assert(parts->children->header_size.physical_size == 45); + test_assert(parts->children->header_size.virtual_size == 45+2); + test_assert(parts->children->body_size.lines == 4); + test_assert(parts->children->body_size.physical_size == 35); + test_assert(parts->children->body_size.virtual_size == 35+4); + test_assert(parts->children->children->flags == (MESSAGE_PART_FLAG_TEXT | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(parts->children->children->physical_pos == 98); + test_assert(parts->children->children->header_size.lines == 2); + test_assert(parts->children->children->header_size.physical_size == 26); + test_assert(parts->children->children->header_size.virtual_size == 26+2); + test_assert(parts->children->children->body_size.lines == 1); + test_assert(parts->children->children->body_size.physical_size == 5); + test_assert(parts->children->children->body_size.virtual_size == 5+1); + + test_parsed_parts(input, parts); + i_stream_unref(&input); + pool_unref(&pool); + test_end(); +} + +static void test_message_parser_garbage_suffix_mime_boundary(void) +{ +static const char input_msg[] = +"Content-Type: multipart/mixed; boundary=\"a\"\n" +"\n" +"--ab\n" +"Content-Type: multipart/mixed; boundary=\"a\"\n" +"\n" +"--ac\n" +"Content-Type: text/plain\n" +"\n" +"body\n"; + struct message_parser_ctx *parser; + struct istream *input; + struct message_part *parts; + struct message_block block; + pool_t pool; + int ret; + + test_begin("message parser garbage suffix mime boundary"); + pool = pool_alloconly_create("message parser", 10240); + input = test_istream_create(input_msg); + + parser = message_parser_init(pool, input, 0, 0); + while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ; + test_assert(ret < 0); + test_assert(message_parser_deinit(&parser, &parts) == 0); + + test_assert(parts->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(parts->header_size.lines == 2); + test_assert(parts->header_size.physical_size == 45); + test_assert(parts->header_size.virtual_size == 45+2); + test_assert(parts->body_size.lines == 7); + test_assert(parts->body_size.physical_size == 86); + test_assert(parts->body_size.virtual_size == 86+7); + test_assert(parts->children->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(parts->children->physical_pos == 50); + test_assert(parts->children->header_size.lines == 2); + test_assert(parts->children->header_size.physical_size == 45); + test_assert(parts->children->header_size.virtual_size == 45+2); + test_assert(parts->children->body_size.lines == 4); + test_assert(parts->children->body_size.physical_size == 36); + test_assert(parts->children->body_size.virtual_size == 36+4); + test_assert(parts->children->children->flags == (MESSAGE_PART_FLAG_TEXT | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(parts->children->children->physical_pos == 100); + test_assert(parts->children->children->header_size.lines == 2); + test_assert(parts->children->children->header_size.physical_size == 26); + test_assert(parts->children->children->header_size.virtual_size == 26+2); + test_assert(parts->children->children->body_size.lines == 1); + test_assert(parts->children->children->body_size.physical_size == 5); + test_assert(parts->children->children->body_size.virtual_size == 5+1); + + test_parsed_parts(input, parts); + i_stream_unref(&input); + pool_unref(&pool); + test_end(); +} + +static void test_message_parser_continuing_mime_boundary(void) +{ +static const char input_msg[] = +"Content-Type: multipart/mixed; boundary=\"a\"\n" +"\n" +"--a\n" +"Content-Type: multipart/mixed; boundary=\"ab\"\n" +"\n" +"--ab\n" +"Content-Type: text/plain\n" +"\n" +"body\n"; + struct message_parser_ctx *parser; + struct istream *input; + struct message_part *parts; + struct message_block block; + pool_t pool; + int ret; + + test_begin("message parser continuing mime boundary"); + pool = pool_alloconly_create("message parser", 10240); + input = test_istream_create(input_msg); + + parser = message_parser_init(pool, input, 0, 0); + while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ; + test_assert(ret < 0); + test_assert(message_parser_deinit(&parser, &parts) == 0); + + test_assert(parts->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(parts->header_size.lines == 2); + test_assert(parts->header_size.physical_size == 45); + test_assert(parts->header_size.virtual_size == 45+2); + test_assert(parts->body_size.lines == 7); + test_assert(parts->body_size.physical_size == 86); + test_assert(parts->body_size.virtual_size == 86+7); + test_assert(parts->children->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(parts->children->physical_pos == 49); + test_assert(parts->children->header_size.lines == 2); + test_assert(parts->children->header_size.physical_size == 46); + test_assert(parts->children->header_size.virtual_size == 46+2); + test_assert(parts->children->body_size.lines == 4); + test_assert(parts->children->body_size.physical_size == 36); + test_assert(parts->children->body_size.virtual_size == 36+4); + test_assert(parts->children->children->flags == (MESSAGE_PART_FLAG_TEXT | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(parts->children->children->physical_pos == 100); + test_assert(parts->children->children->header_size.lines == 2); + test_assert(parts->children->children->header_size.physical_size == 26); + test_assert(parts->children->children->header_size.virtual_size == 26+2); + test_assert(parts->children->children->body_size.lines == 1); + test_assert(parts->children->children->body_size.physical_size == 5); + test_assert(parts->children->children->body_size.virtual_size == 5+1); + + test_parsed_parts(input, parts); + i_stream_unref(&input); + pool_unref(&pool); + test_end(); +} + +static void test_message_parser_continuing_truncated_mime_boundary(void) +{ +static const char input_msg[] = +"Content-Type: multipart/mixed; boundary=\"a\"\n" +"\n" +"--a\n" +"Content-Type: multipart/mixed; boundary=\"ab\"\n" +"MIME-Version: 1.0\n" +"--ab\n" +"Content-Type: text/plain\n" +"\n" +"--ab--\n" +"--a--\n\n"; + struct message_parser_ctx *parser; + struct istream *input; + struct message_part *parts, *part; + struct message_block block; + pool_t pool; + int ret; + + test_begin("message parser continuing truncated mime boundary"); + pool = pool_alloconly_create("message parser", 10240); + input = test_istream_create(input_msg); + + parser = message_parser_init(pool, input, 0, 0); + while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ; + test_assert(ret < 0); + message_parser_deinit(&parser, &parts); + + part = parts; + test_assert(part->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(part->header_size.lines == 2); + test_assert(part->header_size.physical_size == 45); + test_assert(part->header_size.virtual_size == 45+2); + test_assert(part->body_size.lines == 9); + test_assert(part->body_size.physical_size == 112); + test_assert(part->body_size.virtual_size == 112+9); + + part = parts->children; + test_assert(part->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(part->physical_pos == 49); + test_assert(part->header_size.lines == 1); + test_assert(part->header_size.physical_size == 45+17); + test_assert(part->header_size.virtual_size == 45+17+1); + test_assert(part->body_size.lines == 0); + test_assert(part->body_size.physical_size == 0); + test_assert(part->children == NULL); + + /* this will not be a child, since the header was truncated. I guess + we could make it, but it would complicate the message-parser even + more. */ + part = parts->children->next; + test_assert(part->flags == (MESSAGE_PART_FLAG_TEXT | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(part->physical_pos == 117); + test_assert(part->header_size.lines == 1); + test_assert(part->header_size.physical_size == 25); + test_assert(part->header_size.virtual_size == 25+1); + test_assert(part->body_size.lines == 0); + test_assert(part->body_size.physical_size == 0); + test_assert(part->children == NULL); + + part = parts->children->next->next; + test_assert(part->flags == (MESSAGE_PART_FLAG_TEXT | MESSAGE_PART_FLAG_IS_MIME)); + test_assert(part->header_size.lines == 0); + test_assert(part->header_size.physical_size == 0); + test_assert(part->body_size.lines == 0); + test_assert(part->body_size.physical_size == 0); + test_assert(part->children == NULL); + test_assert(part->next == NULL); + + test_parsed_parts(input, parts); i_stream_unref(&input); pool_unref(&pool); test_end(); @@ -218,6 +636,7 @@ static void test_message_parser_no_eoh(void) test_assert(message_parser_parse_next_block(parser, &block) < 0); test_assert(message_parser_deinit(&parser, &parts) == 0); + test_parsed_parts(input, parts); i_stream_unref(&input); pool_unref(&pool); test_end(); @@ -228,6 +647,13 @@ int main(void) static void (*test_functions[])(void) = { test_message_parser_small_blocks, test_message_parser_truncated_mime_headers, + test_message_parser_truncated_mime_headers2, + test_message_parser_truncated_mime_headers3, + test_message_parser_empty_multipart, + test_message_parser_duplicate_mime_boundary, + test_message_parser_garbage_suffix_mime_boundary, + test_message_parser_continuing_mime_boundary, + test_message_parser_continuing_truncated_mime_boundary, test_message_parser_no_eoh, NULL }; diff --git a/src/lib-master/anvil-client.c b/src/lib-master/anvil-client.c index 553b21b041e..eff41ff5780 100644 --- a/src/lib-master/anvil-client.c +++ b/src/lib-master/anvil-client.c @@ -113,7 +113,8 @@ static void anvil_input(struct anvil_client *client) aqueue_delete_tail(client->queries); } if (client->input->stream_errno != 0) { - i_error("read(%s) failed: %m", client->path); + i_error("read(%s) failed: %s", client->path, + i_stream_get_error(client->input)); anvil_reconnect(client); } else if (client->input->eof) { i_error("read(%s) failed: EOF", client->path); diff --git a/src/lib-master/master-instance.c b/src/lib-master/master-instance.c index b05ecb29008..271adfe2782 100644 --- a/src/lib-master/master-instance.c +++ b/src/lib-master/master-instance.c @@ -116,7 +116,7 @@ static int master_instance_list_refresh(struct master_instance_list *list) i_error("Invalid line in %s: %s", list->path, line); } T_END; if (input->stream_errno != 0) { - i_error("read(%s) failed: %m", line); + i_error("read(%s) failed: %s", line, i_stream_get_error(input)); ret = -1; } i_stream_destroy(&input); @@ -147,7 +147,7 @@ master_instance_list_write(struct master_instance_list *list, o_stream_nsend(output, str_data(str), str_len(str)); } if (o_stream_nfinish(output) < 0) { - i_error("write(%s) failed: %m", path); + i_error("write(%s) failed: %s", path, o_stream_get_error(output)); ret = -1; } o_stream_destroy(&output); diff --git a/src/lib-master/master-service-settings.c b/src/lib-master/master-service-settings.c index 5d02c2b80b4..4a25f6592cc 100644 --- a/src/lib-master/master-service-settings.c +++ b/src/lib-master/master-service-settings.c @@ -327,7 +327,8 @@ config_read_reply_header(struct istream *istream, const char *path, pool_t pool, if (ret == 0) return 1; *error_r = istream->stream_errno != 0 ? - t_strdup_printf("read(%s) failed: %m", path) : + t_strdup_printf("read(%s) failed: %s", path, + i_stream_get_error(istream)) : t_strdup_printf("read(%s) failed: EOF", path); return -1; } diff --git a/src/lib-sql/Makefile.am b/src/lib-sql/Makefile.am index 489f257beeb..a6186138d9b 100644 --- a/src/lib-sql/Makefile.am +++ b/src/lib-sql/Makefile.am @@ -36,6 +36,7 @@ sql_drivers = @sql_drivers@ AM_CPPFLAGS = \ -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-settings \ $(SQL_CFLAGS) dist_sources = \ @@ -62,22 +63,22 @@ deplibs = \ if SQL_PLUGINS libdriver_mysql_la_LDFLAGS = -module -avoid-version libdriver_mysql_la_LIBADD = $(MYSQL_LIBS) -libdriver_mysql_la_CPPFLAGS = -I$(top_srcdir)/src/lib $(MYSQL_CFLAGS) +libdriver_mysql_la_CPPFLAGS = $(AM_CPPFLAGS) $(MYSQL_CFLAGS) libdriver_mysql_la_SOURCES = driver-mysql.c libdriver_pgsql_la_LDFLAGS = -module -avoid-version libdriver_pgsql_la_LIBADD = $(PGSQL_LIBS) -libdriver_pgsql_la_CPPFLAGS = -I$(top_srcdir)/src/lib $(PGSQL_CFLAGS) +libdriver_pgsql_la_CPPFLAGS = $(AM_CPPFLAGS) $(PGSQL_CFLAGS) libdriver_pgsql_la_SOURCES = driver-pgsql.c libdriver_sqlite_la_LDFLAGS = -module -avoid-version libdriver_sqlite_la_LIBADD = $(SQLITE_LIBS) -libdriver_sqlite_la_CPPFLAGS = -I$(top_srcdir)/src/lib $(SQLITE_CFLAGS) +libdriver_sqlite_la_CPPFLAGS = $(AM_CPPFLAGS) $(SQLITE_CFLAGS) libdriver_sqlite_la_SOURCES = driver-sqlite.c libdriver_cassandra_la_LDFLAGS = -module -avoid-version libdriver_cassandra_la_LIBADD = $(CASSANDRA_LIBS) -libdriver_cassandra_la_CPPFLAGS = -I$(top_srcdir)/src/lib $(CASSANDRA_CFLAGS) +libdriver_cassandra_la_CPPFLAGS = $(AM_CPPFLAGS) $(CASSANDRA_CFLAGS) libdriver_cassandra_la_SOURCES = driver-cassandra.c sql_libs = diff --git a/src/lib-sql/driver-cassandra.c b/src/lib-sql/driver-cassandra.c index e36edf2a818..78595ceb834 100644 --- a/src/lib-sql/driver-cassandra.c +++ b/src/lib-sql/driver-cassandra.c @@ -8,9 +8,12 @@ #include "net.h" #include "write-full.h" #include "time-util.h" +#include "var-expand.h" +#include "settings-parser.h" #include "sql-api-private.h" #ifdef BUILD_CASSANDRA +#include #include #include @@ -46,11 +49,13 @@ struct cassandra_callback { struct cassandra_db { struct sql_db api; - char *hosts, *keyspace; + char *hosts, *keyspace, *user, *password; CassConsistency read_consistency, write_consistency, delete_consistency; CassConsistency read_fallback_consistency, write_fallback_consistency, delete_fallback_consistency; CassLogLevel log_level; unsigned int protocol_version; + unsigned int num_threads; + unsigned int connect_timeout_secs, request_timeout_secs; in_port_t port; CassCluster *cluster; @@ -63,6 +68,9 @@ struct cassandra_db { ARRAY(struct cassandra_result *) results; unsigned int callback_ids; + char *metrics_path; + struct timeout *to_metrics; + struct timeval first_fallback_sent[CASSANDRA_QUERY_TYPE_COUNT]; time_t last_fallback_warning[CASSANDRA_QUERY_TYPE_COUNT]; unsigned int fallback_failures[CASSANDRA_QUERY_TYPE_COUNT]; @@ -366,7 +374,7 @@ driver_cassandra_escape_string(struct sql_db *db ATTR_UNUSED, static void driver_cassandra_parse_connect_string(struct cassandra_db *db, const char *connect_string) { - const char *const *args, *key, *value; + const char *const *args, *key, *value, *error; string_t *hosts = t_str_new(64); bool read_fallback_set = FALSE, write_fallback_set = FALSE, delete_fallback_set = FALSE; @@ -374,6 +382,8 @@ static void driver_cassandra_parse_connect_string(struct cassandra_db *db, db->read_consistency = CASS_CONSISTENCY_LOCAL_QUORUM; db->write_consistency = CASS_CONSISTENCY_LOCAL_QUORUM; db->delete_consistency = CASS_CONSISTENCY_LOCAL_QUORUM; + db->connect_timeout_secs = SQL_CONNECT_TIMEOUT_SECS; + db->request_timeout_secs = SQL_QUERY_TIMEOUT_SECS; args = t_strsplit_spaces(connect_string, " "); for (; *args != NULL; args++) { @@ -395,6 +405,12 @@ static void driver_cassandra_parse_connect_string(struct cassandra_db *db, strcmp(key, "keyspace") == 0) { i_free(db->keyspace); db->keyspace = i_strdup(value); + } else if (strcmp(key, "user") == 0) { + i_free(db->user); + db->user = i_strdup(value); + } else if (strcmp(key, "password") == 0) { + i_free(db->password); + db->password = i_strdup(value); } else if (strcmp(key, "read_consistency") == 0) { if (consistency_parse(value, &db->read_consistency) < 0) i_fatal("cassandra: Unknown read_consistency: %s", value); @@ -422,6 +438,18 @@ static void driver_cassandra_parse_connect_string(struct cassandra_db *db, } else if (strcmp(key, "version") == 0) { if (str_to_uint(value, &db->protocol_version) < 0) i_fatal("cassandra: Invalid version: %s", value); + } else if (strcmp(key, "num_threads") == 0) { + if (str_to_uint(value, &db->num_threads) < 0) + i_fatal("cassandra: Invalid num_threads: %s", value); + } else if (strcmp(key, "connect_timeout") == 0) { + if (settings_get_time(value, &db->connect_timeout_secs, &error) < 0) + i_fatal("cassandra: Invalid connect_timeout '%s': %s", value, error); + } else if (strcmp(key, "request_timeout") == 0) { + if (settings_get_time(value, &db->request_timeout_secs, &error) < 0) + i_fatal("cassandra: Invalid request_timeout '%s': %s", value, error); + } else if (strcmp(key, "metrics") == 0) { + i_free(db->metrics_path); + db->metrics_path = i_strdup(value); } else { i_fatal("cassandra: Unknown connect string: %s", key); } @@ -441,6 +469,69 @@ static void driver_cassandra_parse_connect_string(struct cassandra_db *db, db->hosts = i_strdup(str_c(hosts)); } +static void +driver_cassandra_get_metrics_json(struct cassandra_db *db, string_t *dest) +{ +#define ADD_UINT64(_struct, _field) \ + str_printfa(dest, "\""#_field"\": %llu,", (unsigned long long)metrics._struct._field); +#define ADD_DOUBLE(_struct, _field) \ + str_printfa(dest, "\""#_field"\": %02lf,", metrics._struct._field); + CassMetrics metrics; + + cass_session_get_metrics(db->session, &metrics); + str_append(dest, "{ \"requests\": {"); + ADD_UINT64(requests, min); + ADD_UINT64(requests, max); + ADD_UINT64(requests, mean); + ADD_UINT64(requests, stddev); + ADD_UINT64(requests, median); + ADD_UINT64(requests, percentile_75th); + ADD_UINT64(requests, percentile_95th); + ADD_UINT64(requests, percentile_98th); + ADD_UINT64(requests, percentile_99th); + ADD_UINT64(requests, percentile_999th); + ADD_DOUBLE(requests, mean_rate); + ADD_DOUBLE(requests, one_minute_rate); + ADD_DOUBLE(requests, five_minute_rate); + ADD_DOUBLE(requests, fifteen_minute_rate); + str_truncate(dest, str_len(dest)-1); + str_append(dest, "}, \"stats\": {"); + ADD_UINT64(stats, total_connections); + ADD_UINT64(stats, available_connections); + ADD_UINT64(stats, exceeded_pending_requests_water_mark); + ADD_UINT64(stats, exceeded_write_bytes_water_mark); + str_truncate(dest, str_len(dest)-1); + str_append(dest, "}, \"errors\": {"); + ADD_UINT64(errors, connection_timeouts); + ADD_UINT64(errors, pending_request_timeouts); + ADD_UINT64(errors, request_timeouts); + str_truncate(dest, str_len(dest)-1); + str_append(dest, "}}"); +} + +static void driver_cassandra_metrics_write(struct cassandra_db *db) +{ + struct var_expand_table tab[] = { + { '\0', NULL, NULL } + }; + string_t *path = t_str_new(64); + string_t *data; + int fd; + + var_expand(path, db->metrics_path, tab); + + fd = open(str_c(path), O_WRONLY | O_CREAT | O_TRUNC | O_NONBLOCK, 0600); + if (fd == -1) { + i_error("creat(%s) failed: %m", str_c(path)); + return; + } + data = t_str_new(1024); + driver_cassandra_get_metrics_json(db, data); + if (write_full(fd, str_data(data), str_len(data)) < 0) + i_error("write(%s) failed: %m", str_c(path)); + i_close_fd(&fd); +} + static struct sql_db *driver_cassandra_init_v(const char *connect_string) { struct cassandra_db *db; @@ -457,14 +548,20 @@ static struct sql_db *driver_cassandra_init_v(const char *connect_string) db->timestamp_gen = cass_timestamp_gen_monotonic_new(); db->cluster = cass_cluster_new(); cass_cluster_set_timestamp_gen(db->cluster, db->timestamp_gen); - cass_cluster_set_connect_timeout(db->cluster, SQL_CONNECT_TIMEOUT_SECS * 1000); - cass_cluster_set_request_timeout(db->cluster, SQL_QUERY_TIMEOUT_SECS * 1000); + cass_cluster_set_connect_timeout(db->cluster, db->connect_timeout_secs * 1000); + cass_cluster_set_request_timeout(db->cluster, db->request_timeout_secs * 1000); cass_cluster_set_contact_points(db->cluster, db->hosts); + if (db->user != NULL && db->password != NULL) + cass_cluster_set_credentials(db->cluster, db->user, db->password); if (db->port != 0) cass_cluster_set_port(db->cluster, db->port); if (db->protocol_version != 0) cass_cluster_set_protocol_version(db->cluster, db->protocol_version); + if (db->num_threads != 0) + cass_cluster_set_num_threads_io(db->cluster, db->num_threads); db->session = cass_session_new(); + if (db->metrics_path != NULL) + db->to_metrics = timeout_add(1000, driver_cassandra_metrics_write, db); i_array_init(&db->results, 16); i_array_init(&db->callbacks, 16); return &db->api; @@ -484,9 +581,14 @@ static void driver_cassandra_deinit_v(struct sql_db *_db) cass_session_free(db->session); cass_cluster_free(db->cluster); cass_timestamp_gen_free(db->timestamp_gen); + if (db->to_metrics != NULL) + timeout_remove(&db->to_metrics); + i_free(db->metrics_path); i_free(db->hosts); i_free(db->error); i_free(db->keyspace); + i_free(db->user); + i_free(db->password); array_free(&_db->module_contexts); i_free(db); } @@ -605,7 +707,16 @@ static void query_callback(CassFuture *future, void *context) result->error = i_strdup_printf("Query '%s' failed: %.*s", result->query, (int)errsize, errmsg); - if (error == CASS_ERROR_SERVER_UNAVAILABLE && + /* unavailable = cassandra server knows that there aren't + enough nodes available. + + write timeout = cassandra server couldn't reach all the + needed nodes. this may be because it hasn't yet detected + that the servers are down, or because the servers are just + too busy. we'll try the fallback consistency to avoid + unnecessary temporary errors. */ + if ((error == CASS_ERROR_SERVER_UNAVAILABLE || + error == CASS_ERROR_SERVER_WRITE_TIMEOUT) && result->fallback_consistency != result->consistency) { /* retry with fallback consistency */ query_resend_with_fallback(result); diff --git a/src/lib-sql/sql-db-cache.c b/src/lib-sql/sql-db-cache.c index d035aaa2240..7b84f229528 100644 --- a/src/lib-sql/sql-db-cache.c +++ b/src/lib-sql/sql-db-cache.c @@ -79,6 +79,7 @@ static void sql_db_cache_free_tail(struct sql_db_cache *cache) i_free(ctx->key); ctx->orig_deinit(db); + i_free(ctx); } static void sql_db_cache_drop_oldest(struct sql_db_cache *cache) diff --git a/src/lib-ssl-iostream/Makefile.am b/src/lib-ssl-iostream/Makefile.am index 7b4ff2c3097..769de87b283 100644 --- a/src/lib-ssl-iostream/Makefile.am +++ b/src/lib-ssl-iostream/Makefile.am @@ -10,8 +10,13 @@ AM_CPPFLAGS = \ if BUILD_OPENSSL module_LTLIBRARIES = libssl_iostream_openssl.la +noinst_LTLIBRARIES += libdovecot_openssl_common.la +libdovecot_openssl_common_la_LIBADD = $(SSL_LIBS) +libdovecot_openssl_common_la_SOURCES = \ + dovecot-openssl-common.c + libssl_iostream_openssl_la_LDFLAGS = -module -avoid-version -libssl_iostream_openssl_la_LIBADD = $(SSL_LIBS) +libssl_iostream_openssl_la_LIBADD = libdovecot_openssl_common.la libssl_iostream_openssl_la_SOURCES = \ iostream-openssl.c \ iostream-openssl-common.c \ @@ -25,6 +30,9 @@ libssl_iostream_la_SOURCES = \ iostream-ssl.c \ $(ssl_sources) +noinst_HEADERS = \ + dovecot-openssl-common.h + headers = \ iostream-openssl.h \ iostream-ssl.h \ diff --git a/src/lib-ssl-iostream/dovecot-openssl-common.c b/src/lib-ssl-iostream/dovecot-openssl-common.c new file mode 100644 index 00000000000..f4dde3233b2 --- /dev/null +++ b/src/lib-ssl-iostream/dovecot-openssl-common.c @@ -0,0 +1,111 @@ +/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dovecot-openssl-common.h" + +#include +#include +#include + +static int openssl_init_refcount = 0; +static ENGINE *dovecot_openssl_engine; + +static void *dovecot_openssl_malloc(size_t size) +{ + /* this may be performance critical, so don't use + i_malloc() or calloc() */ + void *mem = malloc(size); + if (mem == NULL) { + i_fatal_status(FATAL_OUTOFMEM, + "OpenSSL: malloc(%"PRIuSIZE_T"): Out of memory", size); + } + return mem; +} + +static void *dovecot_openssl_realloc(void *ptr, size_t size) +{ + void *mem = realloc(ptr, size); + if (mem == NULL) { + i_fatal_status(FATAL_OUTOFMEM, + "OpenSSL: realloc(%"PRIuSIZE_T"): Out of memory", size); + } + return mem; +} + +void dovecot_openssl_common_global_ref(void) +{ + unsigned char buf; + + if (openssl_init_refcount++ > 0) + return; + + /* use our own memory allocation functions that will die instead of + returning NULL. this avoids random failures on out-of-memory + conditions. */ + if (CRYPTO_set_mem_functions(dovecot_openssl_malloc, + dovecot_openssl_realloc, free) == 0) { + /*i_warning("CRYPTO_set_mem_functions() was called too late");*/ + } + + SSL_library_init(); + SSL_load_error_strings(); + OpenSSL_add_all_algorithms(); + + /* PRNG initialization might want to use /dev/urandom, make sure it + does it before chrooting. We might not have enough entropy at + the first try, so this function may fail. It's still been + initialized though. */ + (void)RAND_bytes(&buf, 1); +} + +bool dovecot_openssl_common_global_unref(void) +{ + i_assert(openssl_init_refcount > 0); + + if (--openssl_init_refcount > 0) + return TRUE; + + if (dovecot_openssl_engine != NULL) { + ENGINE_finish(dovecot_openssl_engine); + dovecot_openssl_engine = NULL; + } +#if OPENSSL_VERSION_NUMBER < 0x10001000L + OBJ_cleanup(); +#endif +#ifdef HAVE_SSL_COMP_FREE_COMPRESSION_METHODS + SSL_COMP_free_compression_methods(); +#endif + ENGINE_cleanup(); + EVP_cleanup(); + CRYPTO_cleanup_all_ex_data(); + ERR_remove_state(0); + ERR_free_strings(); + return FALSE; +} + +int dovecot_openssl_common_global_set_engine(const char *engine, + const char **error_r) +{ + if (dovecot_openssl_engine != NULL) + return 1; + + ENGINE_load_builtin_engines(); + dovecot_openssl_engine = ENGINE_by_id(engine); + if (dovecot_openssl_engine == NULL) { + *error_r = t_strdup_printf("Unknown engine '%s'", engine); + return 0; + } + if (ENGINE_init(dovecot_openssl_engine) == 0) { + *error_r = t_strdup_printf("ENGINE_init(%s) failed", engine); + ENGINE_free(dovecot_openssl_engine); + dovecot_openssl_engine = NULL; + return -1; + } + if (ENGINE_set_default(dovecot_openssl_engine, ENGINE_METHOD_ALL) == 0) { + *error_r = t_strdup_printf("ENGINE_set_default(%s) failed", engine); + ENGINE_free(dovecot_openssl_engine); + dovecot_openssl_engine = NULL; + return -1; + } + return 1; +} diff --git a/src/lib-ssl-iostream/dovecot-openssl-common.h b/src/lib-ssl-iostream/dovecot-openssl-common.h new file mode 100644 index 00000000000..31854d3b1b6 --- /dev/null +++ b/src/lib-ssl-iostream/dovecot-openssl-common.h @@ -0,0 +1,16 @@ +#ifndef DOVECOT_OPENSSL_COMMON_H +#define DOVECOT_OPENSSL_COMMON_H + +/* Initialize OpenSSL if this is the first instance. + Increase initialization reference count. */ +void dovecot_openssl_common_global_ref(void); +/* Deinitialize OpenSSL if this is the last instance. Returns TRUE if there + are more instances left. */ +bool dovecot_openssl_common_global_unref(void); + +/* Set OpenSSL engine if it's not already set. Returns 1 on success, 0 if engine + is unknown, -1 on other error. error_r is set on 0/-1. */ +int dovecot_openssl_common_global_set_engine(const char *engine, + const char **error_r); + +#endif diff --git a/src/lib-ssl-iostream/iostream-openssl-common.c b/src/lib-ssl-iostream/iostream-openssl-common.c index cbdbc4407b3..439ccfd08c7 100644 --- a/src/lib-ssl-iostream/iostream-openssl-common.c +++ b/src/lib-ssl-iostream/iostream-openssl-common.c @@ -1,6 +1,7 @@ /* Copyright (c) 2009-2016 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "str.h" #include "iostream-openssl.h" #include @@ -185,24 +186,36 @@ static const char *ssl_err2str(unsigned long err, const char *data, int flags) const char *openssl_iostream_error(void) { + string_t *errstr = NULL; unsigned long err; - const char *data; + const char *data, *final_error; int flags; while ((err = ERR_get_error_line_data(NULL, NULL, &data, &flags)) != 0) { if (ERR_GET_REASON(err) == ERR_R_MALLOC_FAILURE) i_fatal_status(FATAL_OUTOFMEM, "OpenSSL malloc() failed"); - if (ERR_peek_error() != 0) + if (ERR_peek_error() == 0) break; - i_error("SSL: Stacked error: %s", - ssl_err2str(err, data, flags)); + if (errstr == NULL) + errstr = t_str_new(128); + else + str_append(errstr, ", "); + str_append(errstr, ssl_err2str(err, data, flags)); } if (err == 0) { if (errno != 0) - return strerror(errno); - return "Unknown error"; + final_error = strerror(errno); + else + final_error = "Unknown error"; + } else { + final_error = ssl_err2str(err, data, flags); + } + if (errstr == NULL) + return final_error; + else { + str_printfa(errstr, ", %s", final_error); + return str_c(errstr); } - return ssl_err2str(err, data, flags); } const char *openssl_iostream_key_load_error(void) diff --git a/src/lib-ssl-iostream/iostream-openssl-context.c b/src/lib-ssl-iostream/iostream-openssl-context.c index b976a0c03cd..2b6199df91d 100644 --- a/src/lib-ssl-iostream/iostream-openssl-context.c +++ b/src/lib-ssl-iostream/iostream-openssl-context.c @@ -3,14 +3,13 @@ #include "lib.h" #include "safe-memset.h" #include "iostream-openssl.h" +#include "dovecot-openssl-common.h" #include #include -#include #include #include #include -#include #if !defined(OPENSSL_NO_ECDH) && OPENSSL_VERSION_NUMBER >= 0x10000000L # define HAVE_ECDH @@ -22,7 +21,6 @@ struct ssl_iostream_password_context { }; static bool ssl_global_initialized = FALSE; -static ENGINE *ssl_iostream_engine; int dovecot_ssl_extdata_index; static int ssl_iostream_init_global(const struct ssl_iostream_settings *set, @@ -542,52 +540,36 @@ void openssl_iostream_context_deinit(struct ssl_iostream_context *ctx) void openssl_iostream_global_deinit(void) { - if (ssl_iostream_engine != NULL) - ENGINE_finish(ssl_iostream_engine); - ENGINE_cleanup(); - EVP_cleanup(); - CRYPTO_cleanup_all_ex_data(); - ERR_remove_state(0); - ERR_free_strings(); + dovecot_openssl_common_global_unref(); } static int ssl_iostream_init_global(const struct ssl_iostream_settings *set, const char **error_r) { static char dovecot[] = "dovecot"; - unsigned char buf; + const char *error; if (ssl_global_initialized) return 0; ssl_global_initialized = TRUE; - SSL_library_init(); - SSL_load_error_strings(); - OpenSSL_add_all_algorithms(); + dovecot_openssl_common_global_ref(); dovecot_ssl_extdata_index = SSL_get_ex_new_index(0, dovecot, NULL, NULL, NULL); - /* PRNG initialization might want to use /dev/urandom, make sure it - does it before chrooting. We might not have enough entropy at - the first try, so this function may fail. It's still been - initialized though. */ - (void)RAND_bytes(&buf, 1); - if (set->crypto_device != NULL && *set->crypto_device != '\0') { - ENGINE_load_builtin_engines(); - ssl_iostream_engine = ENGINE_by_id(set->crypto_device); - if (ssl_iostream_engine == NULL) { - *error_r = t_strdup_printf( + switch (dovecot_openssl_common_global_set_engine(set->crypto_device, &error)) { + case 0: + error = t_strdup_printf( "Unknown ssl_crypto_device: %s", set->crypto_device); + /* fall through */ + case -1: + *error_r = error; /* we'll deinit at exit in any case */ return -1; } - ENGINE_init(ssl_iostream_engine); - ENGINE_set_default_RSA(ssl_iostream_engine); - ENGINE_set_default_DSA(ssl_iostream_engine); - ENGINE_set_default_ciphers(ssl_iostream_engine); } return 0; } diff --git a/src/lib-ssl-iostream/istream-openssl.c b/src/lib-ssl-iostream/istream-openssl.c index bdfc21c9dd2..51b6c265990 100644 --- a/src/lib-ssl-iostream/istream-openssl.c +++ b/src/lib-ssl-iostream/istream-openssl.c @@ -33,6 +33,7 @@ static ssize_t i_stream_ssl_read_real(struct istream_private *stream) struct ssl_istream *sstream = (struct ssl_istream *)stream; struct ssl_iostream *ssl_io = sstream->ssl_io; unsigned char buffer[IO_BLOCK_SIZE]; + size_t max_buffer_size = i_stream_get_max_buffer_size(&stream->istream); size_t orig_max_buffer_size = stream->max_buffer_size; size_t size; ssize_t ret, total_ret; @@ -42,9 +43,9 @@ static ssize_t i_stream_ssl_read_real(struct istream_private *stream) return -1; } - if (stream->pos >= stream->max_buffer_size) { + if (stream->pos >= max_buffer_size) { i_stream_compress(stream); - if (stream->pos >= stream->max_buffer_size) + if (stream->pos >= max_buffer_size) return -2; } @@ -62,9 +63,9 @@ static ssize_t i_stream_ssl_read_real(struct istream_private *stream) if (!i_stream_try_alloc(stream, 1, &size)) i_unreached(); - if (stream->pos + size > stream->max_buffer_size) { - i_assert(stream->max_buffer_size > stream->pos); - size = stream->max_buffer_size - stream->pos; + if (stream->pos + size > max_buffer_size) { + i_assert(max_buffer_size > stream->pos); + size = max_buffer_size - stream->pos; } while ((ret = SSL_read(ssl_io->ssl, diff --git a/src/lib-ssl-iostream/ostream-openssl.c b/src/lib-ssl-iostream/ostream-openssl.c index 91fc4d9c994..4a8fe9ecb95 100644 --- a/src/lib-ssl-iostream/ostream-openssl.c +++ b/src/lib-ssl-iostream/ostream-openssl.c @@ -50,7 +50,7 @@ o_stream_ssl_buffer(struct ssl_ostream *sstream, const struct const_iovec *iov, if (sstream->ostream.max_buffer_size == 0) { /* we're requeted to use whatever space is available in the buffer */ - avail = buffer_get_size(sstream->buffer) - sstream->buffer->used; + avail = buffer_get_writable_size(sstream->buffer) - sstream->buffer->used; } else { avail = sstream->ostream.max_buffer_size > sstream->buffer->used ? sstream->ostream.max_buffer_size - sstream->buffer->used : 0; diff --git a/src/lib-stats/stats-connection.c b/src/lib-stats/stats-connection.c index 389ed861c48..740acad7df5 100644 --- a/src/lib-stats/stats-connection.c +++ b/src/lib-stats/stats-connection.c @@ -1,6 +1,7 @@ /* Copyright (c) 2011-2016 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "ioloop.h" #include "str.h" #include "master-service.h" #include "stats-connection.h" @@ -8,6 +9,8 @@ #include #include +#define STATS_EAGAIN_WARN_INTERVAL_SECS 30 + struct stats_connection { int refcount; @@ -15,6 +18,7 @@ struct stats_connection { char *path; bool open_failed; + time_t next_warning_timestamp; }; static bool stats_connection_open(struct stats_connection *conn) @@ -89,7 +93,17 @@ void stats_connection_send(struct stats_connection *conn, const string_t *str) } ret = write(conn->fd, str_data(str), str_len(str)); - if (ret != (ssize_t)str_len(str)) { + if (ret == (ssize_t)str_len(str)) { + /* success */ + } else if (ret < 0 && errno == EAGAIN) { + /* stats process is busy */ + if (ioloop_time > conn->next_warning_timestamp) { + i_warning("write(%s) failed: %m (stats process is busy)", conn->path); + conn->next_warning_timestamp = ioloop_time + + STATS_EAGAIN_WARN_INTERVAL_SECS; + } + } else { + /* error - reconnect */ if (ret < 0) { /* don't log EPIPE errors. they can happen when Dovecot is stopped. */ diff --git a/src/lib-stats/stats.c b/src/lib-stats/stats.c index 258c097f357..30588035e86 100644 --- a/src/lib-stats/stats.c +++ b/src/lib-stats/stats.c @@ -68,8 +68,12 @@ void stats_unregister(struct stats_item **_item) array_delete(&stats_items, idx, 1); i_free(item); - if (array_count(&stats_items) == 0) + if (array_count(&stats_items) == 0) { array_free(&stats_items); + /* all stats should have been freed by now. allow + re-registering and using stats. */ + stats_allocated = FALSE; + } } struct stats *stats_alloc(pool_t pool) diff --git a/src/lib-storage/index/Makefile.am b/src/lib-storage/index/Makefile.am index 6e2fa6f9cd7..220ccd7958a 100644 --- a/src/lib-storage/index/Makefile.am +++ b/src/lib-storage/index/Makefile.am @@ -20,6 +20,7 @@ libstorage_index_la_SOURCES = \ index-mail-binary.c \ index-mail-headers.c \ index-mailbox-size.c \ + index-pop3-uidl.c \ index-rebuild.c \ index-search.c \ index-search-result.c \ @@ -41,6 +42,7 @@ headers = \ index-attachment.h \ index-mail.h \ index-mailbox-size.h \ + index-pop3-uidl.h \ index-rebuild.h \ index-search-private.h \ index-search-result.h \ diff --git a/src/lib-storage/index/cydir/cydir-save.c b/src/lib-storage/index/cydir/cydir-save.c index cc964cd9b03..1d8b696c65d 100644 --- a/src/lib-storage/index/cydir/cydir-save.c +++ b/src/lib-storage/index/cydir/cydir-save.c @@ -162,7 +162,8 @@ static int cydir_save_flush(struct cydir_save_context *ctx, const char *path) int ret = 0; if (o_stream_nfinish(ctx->ctx.data.output) < 0) { - mail_storage_set_critical(storage, "write(%s) failed: %m", path); + mail_storage_set_critical(storage, "write(%s) failed: %s", path, + o_stream_get_error(ctx->ctx.data.output)); ret = -1; } diff --git a/src/lib-storage/index/dbox-common/dbox-mail.c b/src/lib-storage/index/dbox-common/dbox-mail.c index ddd93bdaa61..6785904be13 100644 --- a/src/lib-storage/index/dbox-common/dbox-mail.c +++ b/src/lib-storage/index/dbox-common/dbox-mail.c @@ -5,6 +5,7 @@ #include "str.h" #include "index-storage.h" #include "index-mail.h" +#include "index-pop3-uidl.h" #include "dbox-attachment.h" #include "dbox-storage.h" #include "dbox-file.h" @@ -223,15 +224,31 @@ int dbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field, const char **value_r) { struct dbox_mail *mail = (struct dbox_mail *)_mail; + int ret; /* keep the UIDL in cache file, otherwise POP3 would open all mail files and read the metadata. same for GUIDs if they're used. */ switch (field) { case MAIL_FETCH_UIDL_BACKEND: - return dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_UIDL, - MAIL_CACHE_POP3_UIDL, value_r); + if (!index_pop3_uidl_can_exist(_mail)) { + *value_r = ""; + return 0; + } + ret = dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_UIDL, + MAIL_CACHE_POP3_UIDL, value_r); + if (ret == 0) { + index_pop3_uidl_update_exists(&mail->imail.mail.mail, + (*value_r)[0] != '\0'); + } + return ret; case MAIL_FETCH_POP3_ORDER: + if (!index_pop3_uidl_can_exist(_mail)) { + /* we're assuming that if there's a POP3 order, there's + also a UIDL */ + *value_r = ""; + return 0; + } return dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_ORDER, MAIL_CACHE_POP3_ORDER, value_r); case MAIL_FETCH_GUID: diff --git a/src/lib-storage/index/dbox-common/dbox-save.c b/src/lib-storage/index/dbox-common/dbox-save.c index fe305a6df89..456cfe339c9 100644 --- a/src/lib-storage/index/dbox-common/dbox-save.c +++ b/src/lib-storage/index/dbox-common/dbox-save.c @@ -166,11 +166,15 @@ void dbox_save_write_metadata(struct mail_save_context *_ctx, str_printfa(str, "%c%s\n", DBOX_METADATA_POP3_UIDL, mdata->pop3_uidl); ctx->have_pop3_uidls = TRUE; + ctx->highest_pop3_uidl_seq = + I_MAX(ctx->highest_pop3_uidl_seq, ctx->seq); } if (mdata->pop3_order != 0) { str_printfa(str, "%c%u\n", DBOX_METADATA_POP3_ORDER, mdata->pop3_order); ctx->have_pop3_orders = TRUE; + ctx->highest_pop3_uidl_seq = + I_MAX(ctx->highest_pop3_uidl_seq, ctx->seq); } guid = mdata->guid; diff --git a/src/lib-storage/index/dbox-common/dbox-save.h b/src/lib-storage/index/dbox-common/dbox-save.h index 7b16d1c87ae..9d5f17333c9 100644 --- a/src/lib-storage/index/dbox-common/dbox-save.h +++ b/src/lib-storage/index/dbox-common/dbox-save.h @@ -14,6 +14,7 @@ struct dbox_save_context { struct ostream *dbox_output; + uint32_t highest_pop3_uidl_seq; unsigned int failed:1; unsigned int finished:1; unsigned int have_pop3_uidls:1; diff --git a/src/lib-storage/index/dbox-multi/mdbox-map.c b/src/lib-storage/index/dbox-multi/mdbox-map.c index fce5936d05d..5bc6548bf7c 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-map.c +++ b/src/lib-storage/index/dbox-multi/mdbox-map.c @@ -1086,7 +1086,7 @@ int mdbox_map_append_next(struct mdbox_map_append_context *ctx, ret = 1; existing = TRUE; } else { - ret = mdbox_map_find_appendable_file(ctx, mail_size, flags, + ret = mdbox_map_find_appendable_file(ctx, mail_size, want_altpath, &file_append, output_r); existing = FALSE; } diff --git a/src/lib-storage/index/dbox-multi/mdbox-purge.c b/src/lib-storage/index/dbox-multi/mdbox-purge.c index 8b749f1b073..8921a0b28a1 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-purge.c +++ b/src/lib-storage/index/dbox-multi/mdbox-purge.c @@ -78,7 +78,8 @@ mdbox_file_read_metadata_hdr(struct dbox_file *file, return 0; } mail_storage_set_critical(&file->storage->storage, - "read(%s) failed: %m", file->cur_path); + "read(%s) failed: %s", file->cur_path, + i_stream_get_error(file->input)); return -1; } @@ -192,7 +193,6 @@ mdbox_purge_save_msg(struct mdbox_purge_context *ctx, struct dbox_file *file, enum mdbox_map_append_flags append_flags; uoff_t msg_size; off_t ret; - int read_errno; if (ctx->append_ctx == NULL) ctx->append_ctx = mdbox_map_append_begin(ctx->atomic); @@ -208,19 +208,19 @@ mdbox_purge_save_msg(struct mdbox_purge_context *ctx, struct dbox_file *file, input = i_stream_create_limit(file->input, msg_size); ret = o_stream_send_istream(output, input); - read_errno = input->stream_errno; - i_stream_unref(&input); - - if (read_errno != 0) { - errno = read_errno; + if (input->stream_errno != 0) { mail_storage_set_critical(&file->storage->storage, - "read(%s) failed: %m", file->cur_path); + "read(%s) failed: %s", file->cur_path, + i_stream_get_error(input)); + i_stream_unref(&input); return -1; } + i_stream_unref(&input); if (o_stream_nfinish(output) < 0) { mail_storage_set_critical(&file->storage->storage, - "write(%s) failed: %m", - out_file_append->file->cur_path); + "write(%s) failed: %s", + out_file_append->file->cur_path, + o_stream_get_error(output)); return -1; } if (ret != (off_t)msg_size) { diff --git a/src/lib-storage/index/dbox-multi/mdbox-save.c b/src/lib-storage/index/dbox-multi/mdbox-save.c index 6d1b354f770..d9bdc022b6a 100644 --- a/src/lib-storage/index/dbox-multi/mdbox-save.c +++ b/src/lib-storage/index/dbox-multi/mdbox-save.c @@ -11,6 +11,7 @@ #include "ostream.h" #include "write-full.h" #include "index-mail.h" +#include "index-pop3-uidl.h" #include "mail-copy.h" #include "dbox-save.h" #include "mdbox-storage.h" @@ -216,6 +217,7 @@ static int mdbox_save_finish_write(struct mail_save_context *_ctx) if (ctx->ctx.failed) { mail_index_expunge(ctx->ctx.trans, ctx->ctx.seq); + mail_cache_transaction_reset(ctx->ctx.ctx.transaction->cache_trans); mdbox_map_append_abort(ctx->append_ctx); array_delete(&ctx->mails, array_count(&ctx->mails) - 1, 1); return -1; @@ -325,6 +327,17 @@ int mdbox_transaction_save_commit_pre(struct mail_save_context *_ctx) mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid, &_t->changes->saved_uids); + if (ctx->ctx.highest_pop3_uidl_seq != 0) { + struct seq_range_iter iter; + uint32_t uid; + + seq_range_array_iter_init(&iter, &_t->changes->saved_uids); + if (!seq_range_array_iter_nth(&iter, + ctx->ctx.highest_pop3_uidl_seq-1, &uid)) + i_unreached(); + index_pop3_uidl_set_max_uid(&ctx->mbox->box, ctx->ctx.trans, uid); + } + /* save map UIDs to mailbox index */ if (first_map_uid != 0) mdbox_save_set_map_uids(ctx, first_map_uid, last_map_uid); diff --git a/src/lib-storage/index/dbox-single/sdbox-file.c b/src/lib-storage/index/dbox-single/sdbox-file.c index 4cdaaf2aa79..8c7d7a5632c 100644 --- a/src/lib-storage/index/dbox-single/sdbox-file.c +++ b/src/lib-storage/index/dbox-single/sdbox-file.c @@ -302,6 +302,8 @@ int sdbox_file_move(struct dbox_file *file, bool alt_path) if (dbox_file_is_in_alt(file) == alt_path) return 0; + if (file->alt_path == NULL) + return 0; if (stat(file->cur_path, &st) < 0 && errno == ENOENT) { /* already expunged/moved by another session */ @@ -309,6 +311,9 @@ int sdbox_file_move(struct dbox_file *file, bool alt_path) } dest_path = !alt_path ? file->primary_path : file->alt_path; + + i_assert(dest_path != NULL); + p = strrchr(dest_path, '/'); i_assert(p != NULL); dest_dir = t_strdup_until(dest_path, p); @@ -323,20 +328,15 @@ int sdbox_file_move(struct dbox_file *file, bool alt_path) output = o_stream_create_fd_file(out_fd, 0, FALSE); i_stream_seek(file->input, 0); - while ((ret = o_stream_send_istream(output, file->input)) > 0) ; + ret = o_stream_send_istream(output, file->input) > 0 ? 0 : -1; if (o_stream_nfinish(output) < 0) { - mail_storage_set_critical(storage, "write(%s) failed: %m", - temp_path); + mail_storage_set_critical(storage, "write(%s) failed: %s", + temp_path, o_stream_get_error(output)); ret = -1; } else if (file->input->stream_errno != 0) { - errno = file->input->stream_errno; - dbox_file_set_syscall_error(file, "ftruncate()"); + mail_storage_set_critical(storage, "read(%s) failed: %s", + temp_path, i_stream_get_error(file->input)); ret = -1; - } else if (ret < 0) { - mail_storage_set_critical(storage, - "o_stream_send_istream(%s, %s) " - "failed with unknown error", - temp_path, file->cur_path); } o_stream_unref(&output); diff --git a/src/lib-storage/index/dbox-single/sdbox-save.c b/src/lib-storage/index/dbox-single/sdbox-save.c index 919a628225a..5560d0d7a21 100644 --- a/src/lib-storage/index/dbox-single/sdbox-save.c +++ b/src/lib-storage/index/dbox-single/sdbox-save.c @@ -12,6 +12,7 @@ #include "write-full.h" #include "index-mail.h" #include "mail-copy.h" +#include "index-pop3-uidl.h" #include "dbox-attachment.h" #include "dbox-save.h" #include "sdbox-storage.h" @@ -195,6 +196,7 @@ static int dbox_save_finish_write(struct mail_save_context *_ctx) if (ctx->ctx.failed) { mail_index_expunge(ctx->ctx.trans, ctx->ctx.seq); + mail_cache_transaction_reset(ctx->ctx.ctx.transaction->cache_trans); dbox_file_append_rollback(&ctx->append_ctx); dbox_file_unlink(*files); dbox_file_unref(files); @@ -247,6 +249,10 @@ static int dbox_save_assign_uids(struct sdbox_save_context *ctx, i_assert(ret); if (sdbox_file_assign_uid(sfile, uid) < 0) return -1; + if (ctx->ctx.highest_pop3_uidl_seq == i+1) { + index_pop3_uidl_set_max_uid(&ctx->mbox->box, + ctx->ctx.trans, uid); + } } i_assert(!seq_range_array_iter_nth(&iter, n, &uid)); return 0; diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c index 1923a8f66a2..45be11e3a0e 100644 --- a/src/lib-storage/index/imapc/imapc-mail.c +++ b/src/lib-storage/index/imapc/imapc-mail.c @@ -96,6 +96,26 @@ static int imapc_mail_failed(struct mail *mail, const char *field) return fix_broken_mail ? 0 : -1; } +static uint64_t imapc_mail_get_modseq(struct mail *_mail) +{ + struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box; + struct imapc_msgmap *msgmap; + const uint64_t *modseqs; + unsigned int count; + uint32_t rseq; + + if (!imapc_storage_has_modseqs(mbox->storage)) + return index_mail_get_modseq(_mail); + + msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box); + if (imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq)) { + modseqs = array_get(&mbox->rseq_modseqs, &count); + if (rseq <= count) + return modseqs[rseq-1]; + } + return 1; /* unknown modseq */ +} + static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r) { struct index_mail *mail = (struct index_mail *)_mail; @@ -561,7 +581,7 @@ struct mail_vfuncs imapc_mail_vfuncs = { index_mail_get_flags, index_mail_get_keywords, index_mail_get_keyword_indexes, - index_mail_get_modseq, + imapc_mail_get_modseq, index_mail_get_pvt_modseq, index_mail_get_parts, index_mail_get_date, diff --git a/src/lib-storage/index/imapc/imapc-mailbox.c b/src/lib-storage/index/imapc/imapc-mailbox.c index c3e12d17021..dce673df3dd 100644 --- a/src/lib-storage/index/imapc/imapc-mailbox.c +++ b/src/lib-storage/index/imapc/imapc-mailbox.c @@ -2,6 +2,7 @@ #include "lib.h" #include "ioloop.h" +#include "mail-index-modseq.h" #include "imap-arg.h" #include "imap-seqset.h" #include "imap-util.h" @@ -282,11 +283,12 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply, uint32_t lseq, rseq = reply->num; struct imapc_fetch_request *const *fetch_requestp; struct imapc_mail *const *mailp; - const struct imap_arg *list, *flags_list; + const struct imap_arg *list, *flags_list, *modseq_list; const char *atom, *guid = NULL; const struct mail_index_record *rec = NULL; enum mail_flags flags; uint32_t fetch_uid, uid; + uint64_t modseq = 0; unsigned int i, j; ARRAY_TYPE(const_string) keywords = ARRAY_INIT; bool seen_flags = FALSE, have_labels = FALSE; @@ -319,6 +321,15 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply, array_append(&keywords, &atom, 1); } } + } else if (strcasecmp(atom, "MODSEQ") == 0 && + imapc_storage_has_modseqs(mbox->storage)) { + /* (modseq-number) */ + if (!imap_arg_get_list(&list[i+1], &modseq_list)) + return; + if (!imap_arg_get_atom(&modseq_list[0], &atom) || + str_to_uint64(atom, &modseq) < 0 || + modseq_list[1].type != IMAP_ARG_EOL) + return; } else if (strcasecmp(atom, "X-GM-MSGID") == 0 && !mbox->initial_sync_done) { if (imap_arg_get_atom(&list[i+1], &atom)) @@ -414,6 +425,11 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply, } mail_index_keywords_unref(&kw); } + if (modseq != 0) { + if (mail_index_modseq_lookup(mbox->delayed_sync_view, lseq) < modseq) + mail_index_update_modseq(mbox->delayed_sync_trans, lseq, modseq); + array_idx_set(&mbox->rseq_modseqs, rseq-1, &modseq); + } if (guid != NULL) { struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(&mbox->box); const enum index_cache_field guid_cache_idx = @@ -454,6 +470,8 @@ static void imapc_untagged_expunge(const struct imapc_untagged_reply *reply, } uid = imapc_msgmap_rseq_to_uid(msgmap, rseq); imapc_msgmap_expunge(msgmap, rseq); + if (array_is_created(&mbox->rseq_modseqs)) + array_delete(&mbox->rseq_modseqs, rseq-1, 1); imapc_mailbox_init_delayed_trans(mbox); if (mail_index_lookup_seq(mbox->sync_view, uid, &lseq)) @@ -578,6 +596,19 @@ imapc_resp_text_uidnext(const struct imapc_untagged_reply *reply, mbox->sync_uid_next = uid_next; } +static void +imapc_resp_text_highestmodseq(const struct imapc_untagged_reply *reply, + struct imapc_mailbox *mbox) +{ + uint64_t highestmodseq; + + if (mbox == NULL || + str_to_uint64(reply->resp_text_value, &highestmodseq) < 0) + return; + + mbox->sync_highestmodseq = highestmodseq; +} + static void imapc_resp_text_permanentflags(const struct imapc_untagged_reply *reply, struct imapc_mailbox *mbox) @@ -646,6 +677,8 @@ void imapc_mailbox_register_callbacks(struct imapc_mailbox *mbox) imapc_resp_text_uidvalidity); imapc_mailbox_register_resp_text(mbox, "UIDNEXT", imapc_resp_text_uidnext); + imapc_mailbox_register_resp_text(mbox, "HIGHESTMODSEQ", + imapc_resp_text_highestmodseq); imapc_mailbox_register_resp_text(mbox, "PERMANENTFLAGS", imapc_resp_text_permanentflags); } diff --git a/src/lib-storage/index/imapc/imapc-save.c b/src/lib-storage/index/imapc/imapc-save.c index e6012ff8d8e..89f394c0457 100644 --- a/src/lib-storage/index/imapc/imapc-save.c +++ b/src/lib-storage/index/imapc/imapc-save.c @@ -280,7 +280,8 @@ int imapc_save_finish(struct mail_save_context *_ctx) if (o_stream_nfinish(_ctx->data.output) < 0) { if (!mail_storage_set_error_from_errno(storage)) { mail_storage_set_critical(storage, - "write(%s) failed: %m", ctx->temp_path); + "write(%s) failed: %s", ctx->temp_path, + o_stream_get_error(_ctx->data.output)); } ctx->failed = TRUE; } diff --git a/src/lib-storage/index/imapc/imapc-settings.c b/src/lib-storage/index/imapc/imapc-settings.c index 6325aa8349f..2bbf274fc74 100644 --- a/src/lib-storage/index/imapc/imapc-settings.c +++ b/src/lib-storage/index/imapc/imapc-settings.c @@ -93,6 +93,7 @@ static const struct imapc_feature_list imapc_feature_list[] = { { "proxyauth", IMAPC_FEATURE_PROXYAUTH }, { "fetch-msn-workarounds", IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS }, { "fetch-fix-broken-mails", IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS }, + { "modseq", IMAPC_FEATURE_MODSEQ }, { NULL, 0 } }; diff --git a/src/lib-storage/index/imapc/imapc-settings.h b/src/lib-storage/index/imapc/imapc-settings.h index bb1dec1ae73..fa8be08492f 100644 --- a/src/lib-storage/index/imapc/imapc-settings.h +++ b/src/lib-storage/index/imapc/imapc-settings.h @@ -14,7 +14,8 @@ enum imapc_features { IMAPC_FEATURE_NO_EXAMINE = 0x40, IMAPC_FEATURE_PROXYAUTH = 0x80, IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS = 0x100, - IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS = 0x200 + IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS = 0x200, + IMAPC_FEATURE_MODSEQ = 0x400 }; /* */ diff --git a/src/lib-storage/index/imapc/imapc-storage.c b/src/lib-storage/index/imapc/imapc-storage.c index ebe22b20aca..4c7e1a3eae2 100644 --- a/src/lib-storage/index/imapc/imapc-storage.c +++ b/src/lib-storage/index/imapc/imapc-storage.c @@ -55,6 +55,9 @@ static void imapc_untagged_status(const struct imapc_untagged_reply *reply, struct imapc_storage_client *client); static void imapc_untagged_namespace(const struct imapc_untagged_reply *reply, struct imapc_storage_client *client); +static int imapc_mailbox_run_status(struct mailbox *box, + enum mailbox_status_items items, + struct mailbox_status *status_r); bool imap_resp_text_code_parse(const char *str, enum mail_error *error_r) { @@ -72,6 +75,16 @@ bool imap_resp_text_code_parse(const char *str, enum mail_error *error_r) return FALSE; } +bool imapc_storage_has_modseqs(struct imapc_storage *storage) +{ + enum imapc_capability capa = + imapc_client_get_capabilities(storage->client->client); + + return (capa & (IMAPC_CAPABILITY_CONDSTORE | + IMAPC_CAPABILITY_QRESYNC)) != 0 && + IMAPC_HAS_FEATURE(storage, IMAPC_FEATURE_MODSEQ); +} + static struct mail_storage *imapc_storage_alloc(void) { struct imapc_storage *storage; @@ -603,6 +616,13 @@ static int imapc_mailbox_open(struct mailbox *box) return -1; } + if (imapc_storage_has_modseqs(mbox->storage)) { + if (!array_is_created(&mbox->rseq_modseqs)) + i_array_init(&mbox->rseq_modseqs, 32); + else + array_clear(&mbox->rseq_modseqs); + } + if (imapc_mailbox_select(mbox) < 0) { mailbox_close(box); return -1; @@ -635,6 +655,8 @@ static void imapc_mailbox_close(struct mailbox *box) if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0) mailbox_set_index_error(&mbox->box); } + if (array_is_created(&mbox->rseq_modseqs)) + array_free(&mbox->rseq_modseqs); if (mbox->sync_view != NULL) mail_index_view_close(&mbox->sync_view); if (mbox->to_idle_delay != NULL) @@ -727,6 +749,9 @@ static void imapc_untagged_status(const struct imapc_untagged_reply *reply, status->uidvalidity = num; else if (strcasecmp(key, "UNSEEN") == 0) status->unseen = num; + else if (strcasecmp(key, "HIGHESTMODSEQ") == 0 && + imapc_storage_has_modseqs(storage)) + status->highest_modseq = num; } } @@ -765,15 +790,32 @@ static void imapc_untagged_namespace(const struct imapc_untagged_reply *reply, } } -static void imapc_mailbox_get_selected_status(struct imapc_mailbox *mbox, - enum mailbox_status_items items, - struct mailbox_status *status_r) +static int imapc_mailbox_get_selected_status(struct imapc_mailbox *mbox, + enum mailbox_status_items items, + struct mailbox_status *status_r) { + int ret = 0; + index_storage_get_open_status(&mbox->box, items, status_r); if ((items & STATUS_PERMANENT_FLAGS) != 0) status_r->permanent_flags = mbox->permanent_flags; if ((items & STATUS_FIRST_RECENT_UID) != 0) status_r->first_recent_uid = mbox->highest_nonrecent_uid + 1; + if ((items & STATUS_HIGHESTMODSEQ) != 0) { + /* FIXME: this doesn't work perfectly. we're now just returning + the HIGHESTMODSEQ from the current index, which may or may + not be correct. with QRESYNC enabled we could be returning + sync_highestmodseq, but that would require implementing + VANISHED replies. and without QRESYNC we'd have to issue + STATUS (HIGHESTMODSEQ), which isn't efficient since we get + here constantly (after every IMAP command). */ + } + if (imapc_storage_has_modseqs(mbox->storage)) { + /* even if local indexes are only in memory, we still + have modseqs on the IMAP server itself. */ + status_r->nonpermanent_modseqs = FALSE; + } + return ret; } static int imapc_mailbox_delete(struct mailbox *box) @@ -802,6 +844,9 @@ static int imapc_mailbox_run_status(struct mailbox *box, str_append(str, " UIDVALIDITY"); if ((items & STATUS_UNSEEN) != 0) str_append(str, " UNSEEN"); + if ((items & STATUS_HIGHESTMODSEQ) != 0 && + imapc_storage_has_modseqs(mbox->storage)) + str_append(str, " HIGHESTMODSEQ"); if (str_len(str) == 0) { /* nothing requested */ @@ -833,14 +878,17 @@ static int imapc_mailbox_get_status(struct mailbox *box, status_r->have_guids = TRUE; if (box->opened) { - imapc_mailbox_get_selected_status(mbox, items, status_r); + if (imapc_mailbox_get_selected_status(mbox, items, status_r) < 0) { + /* can't do anything about this */ + } } else if ((items & (STATUS_FIRST_UNSEEN_SEQ | STATUS_KEYWORDS | STATUS_PERMANENT_FLAGS | STATUS_FIRST_RECENT_UID)) != 0) { /* getting these requires opening the mailbox */ if (mailbox_open(box) < 0) return -1; - imapc_mailbox_get_selected_status(mbox, items, status_r); + if (imapc_mailbox_get_selected_status(mbox, items, status_r) < 0) + return -1; } else { if (imapc_mailbox_run_status(box, items, status_r) < 0) return -1; diff --git a/src/lib-storage/index/imapc/imapc-storage.h b/src/lib-storage/index/imapc/imapc-storage.h index 25e398f007a..f2f89aeaa05 100644 --- a/src/lib-storage/index/imapc/imapc-storage.h +++ b/src/lib-storage/index/imapc/imapc-storage.h @@ -108,9 +108,11 @@ struct imapc_mailbox { enum mail_flags permanent_flags; uint32_t highest_nonrecent_uid; + ARRAY(uint64_t) rseq_modseqs; ARRAY_TYPE(uint32_t) delayed_expunged_uids; uint32_t sync_uid_validity; uint32_t sync_uid_next; + uint64_t sync_highestmodseq; uint32_t sync_fetch_first_uid; uint32_t sync_next_lseq; uint32_t sync_next_rseq; @@ -165,6 +167,7 @@ void imapc_mailbox_run_nofetch(struct imapc_mailbox *mbox); void imapc_mail_cache_free(struct imapc_mail_cache *cache); int imapc_mailbox_select(struct imapc_mailbox *mbox); +bool imapc_storage_has_modseqs(struct imapc_storage *storage); bool imap_resp_text_code_parse(const char *str, enum mail_error *error_r); void imapc_copy_error_from_reply(struct imapc_storage *storage, enum mail_error default_error, diff --git a/src/lib-storage/index/imapc/imapc-sync.c b/src/lib-storage/index/imapc/imapc-sync.c index d14b12ccf51..fa3e66f6253 100644 --- a/src/lib-storage/index/imapc/imapc-sync.c +++ b/src/lib-storage/index/imapc/imapc-sync.c @@ -5,6 +5,7 @@ #include "str.h" #include "imap-util.h" #include "mail-cache.h" +#include "mail-index-modseq.h" #include "index-sync-private.h" #include "imapc-client.h" #include "imapc-msgmap.h" @@ -245,6 +246,13 @@ static void imapc_sync_uid_next(struct imapc_sync_context *ctx) } } +static void imapc_sync_highestmodseq(struct imapc_sync_context *ctx) +{ + if (imapc_storage_has_modseqs(ctx->mbox->storage) && + mail_index_modseq_get_highest(ctx->sync_view) < ctx->mbox->sync_highestmodseq) + mail_index_update_highest_modseq(ctx->trans, ctx->mbox->sync_highestmodseq); +} + static void imapc_initial_sync_check(struct imapc_sync_context *ctx, bool nooped) { @@ -311,6 +319,10 @@ imapc_sync_send_commands(struct imapc_sync_context *ctx, uint32_t first_uid) string_t *cmd = t_str_new(64); str_printfa(cmd, "UID FETCH %u:* (FLAGS", first_uid); + if (imapc_storage_has_modseqs(ctx->mbox->storage)) { + str_append(cmd, " MODSEQ"); + mail_index_modseq_enable(ctx->mbox->box.index); + } if (IMAPC_BOX_HAS_FEATURE(ctx->mbox, IMAPC_FEATURE_GMAIL_MIGRATION)) { enum mailbox_info_flags flags; @@ -393,8 +405,9 @@ static void imapc_sync_index(struct imapc_sync_context *ctx) imapc_mailbox_run(mbox); array_free(&ctx->expunged_uids); - /* add uidnext after all appends */ + /* add uidnext & highestmodseq after all appends */ imapc_sync_uid_next(ctx); + imapc_sync_highestmodseq(ctx); if (!ctx->failed) imapc_sync_expunge_eom(ctx); diff --git a/src/lib-storage/index/index-attachment.c b/src/lib-storage/index/index-attachment.c index 5ec3efd042d..4b7abd9129d 100644 --- a/src/lib-storage/index/index-attachment.c +++ b/src/lib-storage/index/index-attachment.c @@ -196,10 +196,9 @@ static int save_check_write_error(struct mail_storage *storage, if (output->last_failed_errno == 0) return 0; - errno = output->last_failed_errno; if (!mail_storage_set_error_from_errno(storage)) { - mail_storage_set_critical(storage, "write(%s) failed: %m", - o_stream_get_name(output)); + mail_storage_set_critical(storage, "write(%s) failed: %s", + o_stream_get_name(output), o_stream_get_error(output)); } return -1; } diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c index dceb2c949bb..1c37e74afa3 100644 --- a/src/lib-storage/index/index-mail.c +++ b/src/lib-storage/index/index-mail.c @@ -1499,6 +1499,8 @@ static void index_mail_close_streams_full(struct index_mail *mail, bool closing) if (data->filter_stream != NULL) i_stream_unref(&data->filter_stream); if (data->stream != NULL) { + struct istream *orig_stream = data->stream; + data->destroying_stream = TRUE; if (!closing && data->destroy_callback_set) { /* we're replacing the stream with a new one. it's @@ -1508,12 +1510,13 @@ static void index_mail_close_streams_full(struct index_mail *mail, bool closing) index_mail_stream_destroy_callback); } i_stream_unref(&data->stream); - if (closing) { - /* there must be no references to the mail when the - mail is being closed. */ - i_assert(!mail->data.destroying_stream); - } else { + /* there must be no references to the mail when the + mail is being closed. */ + if (!closing) data->destroying_stream = FALSE; + else if (mail->data.destroying_stream) { + i_panic("Input stream %s unexpectedly has references", + i_stream_get_name(orig_stream)); } data->initialized_wrapper_stream = FALSE; @@ -2194,6 +2197,7 @@ void index_mail_set_cache_corrupted_reason(struct mail *mail, "Broken %s for mail UID %u in mailbox %s: %s", field_name, mail->uid, mail->box->vname, reason); } + mail_storage_set_internal_error(mail->box->storage); } int index_mail_opened(struct mail *mail ATTR_UNUSED, diff --git a/src/lib-storage/index/index-pop3-uidl.c b/src/lib-storage/index/index-pop3-uidl.c new file mode 100644 index 00000000000..5ebc7073ff6 --- /dev/null +++ b/src/lib-storage/index/index-pop3-uidl.c @@ -0,0 +1,101 @@ +/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "index-storage.h" +#include "index-mail.h" +#include "index-pop3-uidl.h" + +void index_pop3_uidl_set_max_uid(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t uid) +{ + struct mailbox_index_pop3_uidl uidl; + + memset(&uidl, 0, sizeof(uidl)); + uidl.max_uid_with_pop3_uidl = uid; + + mail_index_update_header_ext(trans, box->pop3_uidl_hdr_ext_id, + 0, &uidl, sizeof(uidl)); +} + +bool index_pop3_uidl_can_exist(struct mail *mail) +{ + struct mailbox_index_pop3_uidl uidl; + const void *data; + size_t size; + + /* We'll assume that if the header exists, it's up-to-date. normally + UIDLs are set only during migration, so this value never changes. + Also even if it does, it becomes out-of-date only when the mailbox + is modified with old Dovecot versions. To fix that we'd have to + add and keep updating "max tracked uid" in this header for every + saved mail, which isn't worth it. */ + mail_index_get_header_ext(mail->transaction->view, + mail->box->pop3_uidl_hdr_ext_id, + &data, &size); + if (size < sizeof(uidl)) { + /* this header isn't set yet */ + return TRUE; + } + memcpy(&uidl, data, size); + return mail->uid <= uidl.max_uid_with_pop3_uidl; +} + +void index_pop3_uidl_update_exists(struct mail *mail, bool exists) +{ + struct mailbox_transaction_context *trans = mail->transaction; + + if (exists) { + if (trans->highest_pop3_uidl_uid < mail->uid) { + trans->highest_pop3_uidl_uid = mail->uid; + trans->prev_pop3_uidl_tracking_seq = mail->seq; + } + } else if (mail->seq == trans->prev_pop3_uidl_tracking_seq+1) { + trans->prev_pop3_uidl_tracking_seq++; + } else { + /* skipping mails. we don't know the state. */ + } +} + +void index_pop3_uidl_update_exists_finish(struct mailbox_transaction_context *trans) +{ + struct mail_index_view *view; + struct mailbox_index_pop3_uidl uidl; + const void *data; + size_t size; + bool seen_all_msgs; + + if (trans->highest_pop3_uidl_uid == 0) + return; + + /* First check that we actually looked at UIDL for all messages. + Otherwise we can't say for sure if the newest messages had UIDLs. */ + if (trans->prev_pop3_uidl_tracking_seq != + mail_index_view_get_messages_count(trans->view)) + return; + + /* Just to be sure: Refresh the index and check again. POP3 keeps + transactions open for duration of the entire session. Maybe another + process already added new mails (and already updated this header). + This check is racy, but normally UIDLs aren't added after migration + so it's a bit questionable if it's even worth having this check in + there. */ + view = mail_index_view_open(trans->box->index); + seen_all_msgs = mail_index_refresh(trans->box->index) == 0 && + trans->prev_pop3_uidl_tracking_seq == + mail_index_view_get_messages_count(view); + mail_index_view_close(&view); + if (!seen_all_msgs) + return; + + /* check if we have already the same header */ + mail_index_get_header_ext(trans->view, trans->box->pop3_uidl_hdr_ext_id, + &data, &size); + if (size >= sizeof(uidl)) { + memcpy(&uidl, data, size); + if (trans->highest_pop3_uidl_uid == uidl.max_uid_with_pop3_uidl) + return; + } + index_pop3_uidl_set_max_uid(trans->box, trans->itrans, + trans->highest_pop3_uidl_uid); +} diff --git a/src/lib-storage/index/index-pop3-uidl.h b/src/lib-storage/index/index-pop3-uidl.h new file mode 100644 index 00000000000..956ab7ddd5f --- /dev/null +++ b/src/lib-storage/index/index-pop3-uidl.h @@ -0,0 +1,16 @@ +#ifndef INDEX_POP3_H +#define INDEX_POP3_H + +struct mail_index_transaction; +struct mail; +struct mailbox; +struct mailbox_transaction_context; + +void index_pop3_uidl_set_max_uid(struct mailbox *box, + struct mail_index_transaction *trans, + uint32_t uid); +bool index_pop3_uidl_can_exist(struct mail *mail); +void index_pop3_uidl_update_exists(struct mail *mail, bool exists); +void index_pop3_uidl_update_exists_finish(struct mailbox_transaction_context *trans); + +#endif diff --git a/src/lib-storage/index/index-search.c b/src/lib-storage/index/index-search.c index d874da6c782..46d0f198579 100644 --- a/src/lib-storage/index/index-search.c +++ b/src/lib-storage/index/index-search.c @@ -985,9 +985,6 @@ static bool search_limit_by_hdr(struct index_search_context *ctx, /* UNSEEN with all seen? */ if (args->match_not) return FALSE; - - /* SEEN with all seen */ - args->match_always = TRUE; } else if (args->match_not) { /* UNSEEN with lowwater limiting */ search_limit_lowwater(ctx, @@ -1005,9 +1002,6 @@ static bool search_limit_by_hdr(struct index_search_context *ctx, /* UNDELETED with all deleted? */ if (args->match_not) return FALSE; - - /* DELETED with all deleted */ - args->match_always = TRUE; } else if (!args->match_not) { /* DELETED with lowwater limiting */ search_limit_lowwater(ctx, diff --git a/src/lib-storage/index/index-status.c b/src/lib-storage/index/index-status.c index 0c1c384078b..574339060ef 100644 --- a/src/lib-storage/index/index-status.c +++ b/src/lib-storage/index/index-status.c @@ -264,6 +264,45 @@ static void get_metadata_precache_fields(struct mailbox *box, metadata_r->precache_fields = cache; } +static int +index_mailbox_get_first_save_date(struct mailbox *box, + struct mailbox_metadata *metadata_r) +{ + const struct mail_index_header *hdr; + struct mailbox_transaction_context *t; + struct mail *mail; + uint32_t seq; + int ret = -1; + + hdr = mail_index_get_header(box->view); + if (hdr->messages_count == 0) { + metadata_r->first_save_date = (time_t)-1; + return 0; + } + + t = mailbox_transaction_begin(box, 0); + mail = mail_alloc(t, 0, NULL); + for (seq = 1; seq <= hdr->messages_count; seq++) { + mail_set_seq(mail, seq); + if (mail_get_save_date(mail, &metadata_r->first_save_date) == 0) { + ret = 0; + break; + } + if (mailbox_get_last_mail_error(box) != MAIL_ERROR_EXPUNGED) { + /* failed */ + break; + } + } + mail_free(&mail); + (void)mailbox_transaction_commit(&t); + if (seq > hdr->messages_count) { + /* all messages were expunged after all */ + metadata_r->first_save_date = (time_t)-1; + return 0; + } + return ret; +} + int index_mailbox_get_metadata(struct mailbox *box, enum mailbox_metadata_items items, struct mailbox_metadata *metadata_r) @@ -296,6 +335,10 @@ int index_mailbox_get_metadata(struct mailbox *box, if (index_mailbox_get_physical_size(box, metadata_r) < 0) return -1; } + if ((items & MAILBOX_METADATA_FIRST_SAVE_DATE) != 0) { + if (index_mailbox_get_first_save_date(box, metadata_r) < 0) + return -1; + } if ((items & MAILBOX_METADATA_CACHE_FIELDS) != 0) get_metadata_cache_fields(box, metadata_r); if ((items & MAILBOX_METADATA_PRECACHE_FIELDS) != 0) diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c index 339ebbb7d10..0d5a407ad5c 100644 --- a/src/lib-storage/index/index-storage.c +++ b/src/lib-storage/index/index-storage.c @@ -293,6 +293,9 @@ int index_storage_mailbox_open(struct mailbox *box, bool move_to_memory) mail_index_ext_register(box->index, "hdr-vsize", sizeof(struct mailbox_index_vsize), 0, sizeof(uint64_t)); + box->pop3_uidl_hdr_ext_id = + mail_index_ext_register(box->index, "hdr-pop3-uidl", + sizeof(struct mailbox_index_pop3_uidl), 0, 0); box->opened = TRUE; diff --git a/src/lib-storage/index/index-storage.h b/src/lib-storage/index/index-storage.h index 24bf9f2d17e..be43e1d78c3 100644 --- a/src/lib-storage/index/index-storage.h +++ b/src/lib-storage/index/index-storage.h @@ -18,6 +18,16 @@ enum mailbox_lock_notify_type { MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE }; +enum index_storage_list_change { + INDEX_STORAGE_LIST_CHANGE_ERROR = -1, + INDEX_STORAGE_LIST_CHANGE_NONE = 0, + INDEX_STORAGE_LIST_CHANGE_INMEMORY, + INDEX_STORAGE_LIST_CHANGE_NORECORD, + INDEX_STORAGE_LIST_CHANGE_NOT_IN_FS, + INDEX_STORAGE_LIST_CHANGE_SIZE_CHANGED, + INDEX_STORAGE_LIST_CHANGE_MTIME_CHANGED +}; + struct index_mailbox_context { union mailbox_module_context module_ctx; enum mail_index_open_flags index_flags; @@ -155,6 +165,10 @@ bool index_keyword_array_cmp(const ARRAY_TYPE(keyword_indexes) *k1, int index_storage_list_index_has_changed(struct mailbox *box, struct mail_index_view *list_view, uint32_t seq); +enum index_storage_list_change +index_storage_list_index_has_changed_full(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq); void index_storage_list_index_update_sync(struct mailbox *box, struct mail_index_transaction *trans, uint32_t seq); diff --git a/src/lib-storage/index/index-sync.c b/src/lib-storage/index/index-sync.c index 79c44329c28..8b47937ae5e 100644 --- a/src/lib-storage/index/index-sync.c +++ b/src/lib-storage/index/index-sync.c @@ -291,6 +291,17 @@ void index_sync_update_recent_count(struct mailbox *box) } } +static void index_mailbox_sync_free(struct index_mailbox_sync_context *ctx) +{ + if (array_is_created(&ctx->flag_updates)) + array_free(&ctx->flag_updates); + if (array_is_created(&ctx->hidden_updates)) + array_free(&ctx->hidden_updates); + if (array_is_created(&ctx->all_flag_update_uids)) + array_free(&ctx->all_flag_update_uids); + i_free(ctx); +} + int index_mailbox_sync_deinit(struct mailbox_sync_context *_ctx, struct mailbox_sync_status *status_r) { @@ -314,6 +325,10 @@ int index_mailbox_sync_deinit(struct mailbox_sync_context *_ctx, ret = -1; } } + if (ret < 0) { + index_mailbox_sync_free(ctx); + return -1; + } index_mailbox_expunge_unseen_recent(ctx); if ((_ctx->box->flags & MAILBOX_FLAG_DROP_RECENT) == 0 && @@ -327,18 +342,10 @@ int index_mailbox_sync_deinit(struct mailbox_sync_context *_ctx, /* update search results after private index is updated */ index_sync_search_results_update(ctx); - - if (array_is_created(&ctx->flag_updates)) - array_free(&ctx->flag_updates); - if (array_is_created(&ctx->hidden_updates)) - array_free(&ctx->hidden_updates); - if (array_is_created(&ctx->all_flag_update_uids)) - array_free(&ctx->all_flag_update_uids); - /* update vsize header if wanted */ - if (ret == 0) - index_mailbox_vsize_update_appends(_ctx->box); - i_free(ctx); + index_mailbox_vsize_update_appends(_ctx->box); + + index_mailbox_sync_free(ctx); return ret; } @@ -403,9 +410,10 @@ index_list_get_ext_id(struct mailbox *box, struct mail_index_view *view) return ibox->list_index_sync_ext_id; } -int index_storage_list_index_has_changed(struct mailbox *box, - struct mail_index_view *list_view, - uint32_t seq) +enum index_storage_list_change +index_storage_list_index_has_changed_full(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq) { const struct index_storage_list_index_record *rec; const void *data; @@ -416,7 +424,7 @@ int index_storage_list_index_has_changed(struct mailbox *box, int ret; if (mail_index_is_in_memory(mail_index_view_get_index(list_view))) - return 1; + return INDEX_STORAGE_LIST_CHANGE_INMEMORY; ext_id = index_list_get_ext_id(box, list_view); mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged); @@ -424,28 +432,43 @@ int index_storage_list_index_has_changed(struct mailbox *box, if (rec == NULL || expunged || rec->size == 0 || rec->mtime == 0) { /* doesn't exist / not synced */ - return 1; + return INDEX_STORAGE_LIST_CHANGE_NORECORD; } if (box->storage->set->mailbox_list_index_very_dirty_syncs) - return 0; + return INDEX_STORAGE_LIST_CHANGE_NONE; ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &dir); if (ret < 0) - return -1; + return INDEX_STORAGE_LIST_CHANGE_ERROR; i_assert(ret > 0); path = t_strconcat(dir, "/", box->index_prefix, ".log", NULL); if (stat(path, &st) < 0) { if (errno == ENOENT) - return 1; + return INDEX_STORAGE_LIST_CHANGE_NOT_IN_FS; mail_storage_set_critical(box->storage, "stat(%s) failed: %m", path); - return -1; + return INDEX_STORAGE_LIST_CHANGE_ERROR; } - if (rec->size != (st.st_size & 0xffffffffU) || - rec->mtime != (st.st_mtime & 0xffffffffU)) + if (rec->size != (st.st_size & 0xffffffffU)) + return INDEX_STORAGE_LIST_CHANGE_SIZE_CHANGED; + if (rec->mtime != (st.st_mtime & 0xffffffffU)) + return INDEX_STORAGE_LIST_CHANGE_MTIME_CHANGED; + return INDEX_STORAGE_LIST_CHANGE_NONE; +} + +int index_storage_list_index_has_changed(struct mailbox *box, + struct mail_index_view *list_view, + uint32_t seq) +{ + switch (index_storage_list_index_has_changed_full(box, list_view, seq)) { + case INDEX_STORAGE_LIST_CHANGE_ERROR: + return -1; + case INDEX_STORAGE_LIST_CHANGE_NONE: + return 0; + default: return 1; - return 0; + } } void index_storage_list_index_update_sync(struct mailbox *box, diff --git a/src/lib-storage/index/index-transaction.c b/src/lib-storage/index/index-transaction.c index 4c7b42cc3b6..db1f21914e6 100644 --- a/src/lib-storage/index/index-transaction.c +++ b/src/lib-storage/index/index-transaction.c @@ -5,6 +5,7 @@ #include "dict.h" #include "index-storage.h" #include "index-sync-private.h" +#include "index-pop3-uidl.h" #include "index-mail.h" static void index_transaction_free(struct mailbox_transaction_context *t) @@ -28,6 +29,7 @@ index_transaction_index_commit(struct mail_index_transaction *index_trans, struct index_mailbox_sync_pvt_context *pvt_sync_ctx = NULL; int ret = 0; + index_pop3_uidl_update_exists_finish(t); if (t->nontransactional_changes) t->changes->changed = TRUE; diff --git a/src/lib-storage/index/maildir/maildir-keywords.c b/src/lib-storage/index/maildir/maildir-keywords.c index bc8d34a7386..161bd01286e 100644 --- a/src/lib-storage/index/maildir/maildir-keywords.c +++ b/src/lib-storage/index/maildir/maildir-keywords.c @@ -168,7 +168,8 @@ static int maildir_keywords_sync(struct maildir_keywords *mk) *p++ = '\0'; if (str_to_uint(line, &idx) < 0 || - idx >= MAILDIR_MAX_KEYWORDS || *p == '\0') { + idx >= MAILDIR_MAX_KEYWORDS || *p == '\0' || + hash_table_lookup(mk->hash, p) != NULL) { /* shouldn't happen */ continue; } diff --git a/src/lib-storage/index/maildir/maildir-save.c b/src/lib-storage/index/maildir/maildir-save.c index 228c2721425..2ee201df9ec 100644 --- a/src/lib-storage/index/maildir/maildir-save.c +++ b/src/lib-storage/index/maildir/maildir-save.c @@ -536,7 +536,7 @@ static int maildir_save_finish_real(struct mail_save_context *_ctx) { struct maildir_save_context *ctx = (struct maildir_save_context *)_ctx; struct mail_storage *storage = &ctx->mbox->storage->storage; - const char *path; + const char *path, *output_errstr; off_t real_size; uoff_t size; int output_errno; @@ -551,7 +551,8 @@ static int maildir_save_finish_real(struct mail_save_context *_ctx) if (!ctx->failed && o_stream_nfinish(_ctx->data.output) < 0) { if (!mail_storage_set_error_from_errno(storage)) { mail_storage_set_critical(storage, - "write(%s) failed: %m", path); + "write(%s) failed: %s", path, + o_stream_get_error(_ctx->data.output)); } ctx->failed = TRUE; } @@ -582,6 +583,7 @@ static int maildir_save_finish_real(struct mail_save_context *_ctx) ctx->file_last->vsize = (uoff_t)-1; output_errno = _ctx->data.output->last_failed_errno; + output_errstr = t_strdup(o_stream_get_error(_ctx->data.output)); o_stream_destroy(&_ctx->data.output); if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER && @@ -625,13 +627,12 @@ static int maildir_save_finish_real(struct mail_save_context *_ctx) /* delete the tmp file */ i_unlink_if_exists(path); - errno = output_errno; - if (ENOQUOTA(errno)) { + if (ENOQUOTA(output_errno)) { mail_storage_set_error(storage, MAIL_ERROR_NOQUOTA, MAIL_ERRSTR_NO_QUOTA); - } else if (errno != 0) { + } else if (output_errno != 0) { mail_storage_set_critical(storage, - "write(%s) failed: %m", path); + "write(%s) failed: %s", path, output_errstr); } maildir_save_remove_last_filename(ctx); diff --git a/src/lib-storage/index/maildir/maildir-sync.c b/src/lib-storage/index/maildir/maildir-sync.c index 1b80e41638f..9ad66093eb1 100644 --- a/src/lib-storage/index/maildir/maildir-sync.c +++ b/src/lib-storage/index/maildir/maildir-sync.c @@ -972,8 +972,9 @@ maildir_sync_context(struct maildir_sync_context *ctx, bool forced, /* UID is expunged */ *find_uid = 0; } else if ((flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) == 0) { - /* we didn't find it, possibly expunged? */ *find_uid = 0; + } else { + /* we didn't find it, possibly expunged? */ } } diff --git a/src/lib-storage/index/maildir/maildir-uidlist.c b/src/lib-storage/index/maildir/maildir-uidlist.c index eb89aa558d4..bed17e5d2e5 100644 --- a/src/lib-storage/index/maildir/maildir-uidlist.c +++ b/src/lib-storage/index/maildir/maildir-uidlist.c @@ -832,9 +832,9 @@ maildir_uidlist_update_read(struct maildir_uidlist *uidlist, if (input->stream_errno == ESTALE && try_retry) *retry_r = TRUE; else { - errno = input->stream_errno; mail_storage_set_critical(storage, - "read(%s) failed: %m", uidlist->path); + "read(%s) failed: %s", uidlist->path, + i_stream_get_error(input)); } uidlist->last_read_offset = 0; } @@ -1312,7 +1312,8 @@ static int maildir_uidlist_write_fd(struct maildir_uidlist *uidlist, int fd, maildir_uidlist_iter_deinit(&iter); if (o_stream_nfinish(output) < 0) { - mail_storage_set_critical(storage, "write(%s) failed: %m", path); + mail_storage_set_critical(storage, "write(%s) failed: %s", path, + o_stream_get_error(output)); o_stream_unref(&output); return -1; } @@ -1700,8 +1701,13 @@ maildir_uidlist_sync_next_partial(struct maildir_uidlist_sync_ctx *ctx, rec = p_new(uidlist->record_pool, struct maildir_uidlist_rec, 1); rec->uid = (uint32_t)-1; + rec->filename = p_strdup(uidlist->record_pool, filename); array_append(&uidlist->records, &rec, 1); uidlist->change_counter++; + + hash_table_insert(uidlist->files, rec->filename, rec); + } else if (strcmp(rec->filename, filename) != 0) { + rec->filename = p_strdup(uidlist->record_pool, filename); } if (uid != 0) { if (rec->uid != uid && rec->uid != (uint32_t)-1) { @@ -1723,8 +1729,6 @@ maildir_uidlist_sync_next_partial(struct maildir_uidlist_sync_ctx *ctx, rec->flags &= ~MAILDIR_UIDLIST_REC_FLAG_NEW_DIR; rec->flags = (rec->flags | flags) & ~MAILDIR_UIDLIST_REC_FLAG_NONSYNCED; - rec->filename = p_strdup(uidlist->record_pool, filename); - hash_table_insert(uidlist->files, rec->filename, rec); ctx->finished = FALSE; *rec_r = rec; @@ -1798,6 +1802,8 @@ int maildir_uidlist_sync_next_uid(struct maildir_uidlist_sync_ctx *ctx, to check for duplicates. */ rec->flags &= ~(MAILDIR_UIDLIST_REC_FLAG_NEW_DIR | MAILDIR_UIDLIST_REC_FLAG_MOVED); + if (strcmp(rec->filename, filename) != 0) + rec->filename = p_strdup(ctx->record_pool, filename); } else { old_rec = hash_table_lookup(uidlist->files, filename); i_assert(old_rec != NULL || UIDLIST_IS_LOCKED(uidlist)); @@ -1815,6 +1821,8 @@ int maildir_uidlist_sync_next_uid(struct maildir_uidlist_sync_ctx *ctx, /* didn't exist in uidlist, it's recent */ flags |= MAILDIR_UIDLIST_REC_FLAG_RECENT; } + rec->filename = p_strdup(ctx->record_pool, filename); + hash_table_insert(ctx->files, rec->filename, rec); array_append(&ctx->records, &rec, 1); } @@ -1825,8 +1833,6 @@ int maildir_uidlist_sync_next_uid(struct maildir_uidlist_sync_ctx *ctx, } rec->flags = (rec->flags | flags) & ~MAILDIR_UIDLIST_REC_FLAG_NONSYNCED; - rec->filename = p_strdup(ctx->record_pool, filename); - hash_table_insert(ctx->files, rec->filename, rec); *rec_r = rec; return 1; } diff --git a/src/lib-storage/index/maildir/maildir-util.c b/src/lib-storage/index/maildir/maildir-util.c index e527989792c..e3b76051993 100644 --- a/src/lib-storage/index/maildir/maildir-util.c +++ b/src/lib-storage/index/maildir/maildir-util.c @@ -117,18 +117,26 @@ static int maildir_file_do_try(struct maildir_mailbox *mbox, uint32_t uid, } static int do_racecheck(struct maildir_mailbox *mbox, const char *path, - void *context ATTR_UNUSED) + void *context) { + const uint32_t *uidp = context; struct stat st; + int ret; - if (lstat(path, &st) == 0 && (st.st_mode & S_IFMT) == S_IFLNK) { + ret = lstat(path, &st); + if (ret == 0 && (st.st_mode & S_IFMT) == S_IFLNK) { /* most likely a symlink pointing to a nonexistent file */ mail_storage_set_critical(&mbox->storage->storage, - "Maildir: Symlink destination doesn't exist: %s", path); + "Maildir: Symlink destination doesn't exist for UID=%u: %s", *uidp, path); return -2; + } else if (ret < 0 && errno != ENOENT) { + mail_storage_set_critical(&mbox->storage->storage, + "lstat(%s) failed: %m", path); + return -1; } else { + /* success or ENOENT, either way we're done */ mail_storage_set_critical(&mbox->storage->storage, - "maildir_file_do(%s): Filename keeps changing", path); + "maildir_file_do(%s): Filename keeps changing for UID=%u", path, *uidp); return -1; } } @@ -160,7 +168,7 @@ int maildir_file_do(struct maildir_mailbox *mbox, uint32_t uid, } if (i == MAILDIR_RESYNC_RETRY_COUNT) T_BEGIN { - ret = maildir_file_do_try(mbox, uid, do_racecheck, context); + ret = maildir_file_do_try(mbox, uid, do_racecheck, &uid); } T_END; return ret == -2 ? 0 : ret; diff --git a/src/lib-storage/index/mbox/mbox-lock.c b/src/lib-storage/index/mbox/mbox-lock.c index 647f33611f6..9b6ff4ffc45 100644 --- a/src/lib-storage/index/mbox/mbox-lock.c +++ b/src/lib-storage/index/mbox/mbox-lock.c @@ -41,7 +41,7 @@ enum mbox_dotlock_op { struct mbox_lock_context { struct mbox_mailbox *mbox; - int lock_status[MBOX_LOCK_COUNT]; + bool locked_status[MBOX_LOCK_COUNT]; bool checked_file; int lock_type; @@ -674,7 +674,8 @@ mbox_lock_list(struct mbox_lock_context *ctx, int lock_type, { enum mbox_lock_type *lock_types; enum mbox_lock_type type; - int i, ret = 0, lock_status; + int i, ret = 0; + bool locked_status; ctx->lock_type = lock_type; @@ -684,11 +685,11 @@ mbox_lock_list(struct mbox_lock_context *ctx, int lock_type, ctx->mbox->storage->read_locks; for (i = idx; lock_types[i] != (enum mbox_lock_type)-1; i++) { type = lock_types[i]; - lock_status = lock_type != F_UNLCK; + locked_status = lock_type != F_UNLCK; - if (ctx->lock_status[type] == lock_status) + if (ctx->locked_status[type] == locked_status) continue; - ctx->lock_status[type] = lock_status; + ctx->locked_status[type] = locked_status; ret = lock_data[type].func(ctx, lock_type, max_wait_time); if (ret <= 0) @@ -733,9 +734,9 @@ static int mbox_update_locking(struct mbox_mailbox *mbox, int lock_type, mbox->storage->read_locks; for (i = 0; i < MBOX_LOCK_COUNT; i++) - ctx.lock_status[i] = 1; + ctx.locked_status[i] = TRUE; for (i = 0; read_locks[i] != (enum mbox_lock_type)-1; i++) - ctx.lock_status[read_locks[i]] = 0; + ctx.locked_status[read_locks[i]] = FALSE; drop_locks = TRUE; } else { drop_locks = FALSE; @@ -761,11 +762,11 @@ static int mbox_update_locking(struct mbox_mailbox *mbox, int lock_type, const enum mbox_lock_type *write_locks = mbox->storage->write_locks; - memset(ctx.lock_status, 0, sizeof(ctx.lock_status)); + memset(ctx.locked_status, 0, sizeof(ctx.locked_status)); for (i = 0; write_locks[i] != (enum mbox_lock_type)-1; i++) - ctx.lock_status[write_locks[i]] = 1; + ctx.locked_status[write_locks[i]] = TRUE; for (i = 0; read_locks[i] != (enum mbox_lock_type)-1; i++) - ctx.lock_status[read_locks[i]] = 0; + ctx.locked_status[read_locks[i]] = FALSE; mbox->mbox_lock_type = F_WRLCK; mbox_lock_list(&ctx, F_UNLCK, 0, 0); @@ -877,7 +878,7 @@ int mbox_unlock(struct mbox_mailbox *mbox, unsigned int lock_id) ctx.mbox = mbox; for (i = 0; i < MBOX_LOCK_COUNT; i++) - ctx.lock_status[i] = 1; + ctx.locked_status[i] = TRUE; return mbox_unlock_files(&ctx); } diff --git a/src/lib-storage/index/mbox/mbox-md5-all.c b/src/lib-storage/index/mbox/mbox-md5-all.c index e9831ffbf37..c8e744ef803 100644 --- a/src/lib-storage/index/mbox/mbox-md5-all.c +++ b/src/lib-storage/index/mbox/mbox-md5-all.c @@ -26,7 +26,7 @@ static void mbox_md5_all_more(struct mbox_md5_context *ctx, } static void mbox_md5_all_finish(struct mbox_md5_context *ctx, - unsigned char result[16]) + unsigned char result[STATIC_ARRAY 16]) { md5_final(&ctx->hdr_md5_ctx, result); i_free(ctx); diff --git a/src/lib-storage/index/mbox/mbox-md5-apop3d.c b/src/lib-storage/index/mbox/mbox-md5-apop3d.c index e980d0b5961..7786a6c195a 100644 --- a/src/lib-storage/index/mbox/mbox-md5-apop3d.c +++ b/src/lib-storage/index/mbox/mbox-md5-apop3d.c @@ -106,7 +106,7 @@ static void mbox_md5_apop3d_more(struct mbox_md5_context *ctx, } static void mbox_md5_apop3d_finish(struct mbox_md5_context *ctx, - unsigned char result[16]) + unsigned char result[STATIC_ARRAY 16]) { md5_final(&ctx->hdr_md5_ctx, result); i_free(ctx); diff --git a/src/lib-storage/index/mbox/mbox-md5.h b/src/lib-storage/index/mbox/mbox-md5.h index debd2101a32..7584052e64b 100644 --- a/src/lib-storage/index/mbox/mbox-md5.h +++ b/src/lib-storage/index/mbox/mbox-md5.h @@ -8,7 +8,7 @@ struct mbox_md5_vfuncs { void (*more)(struct mbox_md5_context *ctx, struct message_header_line *hdr); void (*finish)(struct mbox_md5_context *ctx, - unsigned char result[16]); + unsigned char result[STATIC_ARRAY 16]); }; extern struct mbox_md5_vfuncs mbox_md5_apop3d; diff --git a/src/lib-storage/index/mbox/mbox-sync-rewrite.c b/src/lib-storage/index/mbox/mbox-sync-rewrite.c index ef902441a7d..63bb4c125cc 100644 --- a/src/lib-storage/index/mbox/mbox-sync-rewrite.c +++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c @@ -354,7 +354,7 @@ static int mbox_sync_read_next(struct mbox_sync_context *sync_ctx, } first_mail_expunge_extra = 1 + - sync_ctx->first_mail_crlf_expunged ? 1 : 0; + (sync_ctx->first_mail_crlf_expunged ? 1 : 0); if (mails[idx].from_offset + first_mail_expunge_extra - expunged_space != 0) { sync_ctx->dest_first_mail = mails[idx].from_offset == 0; diff --git a/src/lib-storage/list/mailbox-list-index-backend.c b/src/lib-storage/list/mailbox-list-index-backend.c index da40d0aa3c4..4cef1bcf693 100644 --- a/src/lib-storage/list/mailbox-list-index-backend.c +++ b/src/lib-storage/list/mailbox-list-index-backend.c @@ -526,7 +526,8 @@ index_list_delete_mailbox(struct mailbox_list *_list, const char *name) if (ret <= 0) return ret; - if ((_list->flags & MAILBOX_LIST_FLAG_NO_MAIL_FILES) != 0) { + if ((_list->flags & (MAILBOX_LIST_FLAG_NO_MAIL_FILES | + MAILBOX_LIST_FLAG_NO_DELETES)) != 0) { ret = 0; } else if ((_list->flags & MAILBOX_LIST_FLAG_MAILBOX_FILES) != 0) { ret = mailbox_list_delete_mailbox_file(_list, name, path); @@ -535,7 +536,8 @@ index_list_delete_mailbox(struct mailbox_list *_list, const char *name) path, TRUE); } - if (ret == 0 || (_list->props & MAILBOX_LIST_PROP_AUTOCREATE_DIRS) != 0) + if ((ret == 0 || (_list->props & MAILBOX_LIST_PROP_AUTOCREATE_DIRS) != 0) && + (_list->flags & MAILBOX_LIST_FLAG_NO_DELETES) == 0) index_list_delete_finish(_list, name); if (ret == 0) { if (index_list_delete_entry(list, name, TRUE) < 0) diff --git a/src/lib-storage/list/mailbox-list-index-iter.c b/src/lib-storage/list/mailbox-list-index-iter.c index 08e02827b42..cbaf435032b 100644 --- a/src/lib-storage/list/mailbox-list-index-iter.c +++ b/src/lib-storage/list/mailbox-list-index-iter.c @@ -101,10 +101,12 @@ mailbox_list_index_update_info(struct mailbox_list_index_iterate_context *ctx) &ctx->info.flags); } - box = mailbox_alloc(ctx->ctx.list, ctx->info.vname, 0); - mailbox_list_index_status_set_info_flags(box, node->uid, - &ctx->info.flags); - mailbox_free(&box); + if ((ctx->ctx.flags & MAILBOX_LIST_ITER_RETURN_NO_FLAGS) == 0) { + box = mailbox_alloc(ctx->ctx.list, ctx->info.vname, 0); + mailbox_list_index_status_set_info_flags(box, node->uid, + &ctx->info.flags); + mailbox_free(&box); + } } static void diff --git a/src/lib-storage/list/mailbox-list-index-status.c b/src/lib-storage/list/mailbox-list-index-status.c index 8b65943dfa8..82f96433f04 100644 --- a/src/lib-storage/list/mailbox-list-index-status.c +++ b/src/lib-storage/list/mailbox-list-index-status.c @@ -34,8 +34,8 @@ struct index_list_storage_module index_list_storage_module = ((box)->inbox_any) static int -index_list_open_view(struct mailbox *box, struct mail_index_view **view_r, - uint32_t *seq_r) +index_list_open_view(struct mailbox *box, bool refresh, + struct mail_index_view **view_r, uint32_t *seq_r) { struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(box->list); struct mailbox_list_index_node *node; @@ -58,6 +58,9 @@ index_list_open_view(struct mailbox *box, struct mail_index_view **view_r, if (!mail_index_lookup_seq(view, node->uid, &seq)) { /* our in-memory tree is out of sync */ ret = 1; + } else if (!refresh) { + /* this operation doesn't need the index to be up-to-date */ + ret = 0; } else T_BEGIN { ret = box->v.list_index_has_changed == NULL ? 0 : box->v.list_index_has_changed(box, view, seq); @@ -89,7 +92,7 @@ index_list_exists(struct mailbox *box, bool auto_boxes, uint32_t seq; int ret; - if ((ret = index_list_open_view(box, &view, &seq)) <= 0) { + if ((ret = index_list_open_view(box, FALSE, &view, &seq)) <= 0) { /* failure / not found. fallback to the real storage check just in case to see if the mailbox was just created. */ return ibox->module_ctx.super. @@ -186,6 +189,9 @@ index_list_get_cached_status(struct mailbox *box, uint32_t seq; int ret; + if (items == 0) + return 1; + if ((items & STATUS_UNSEEN) != 0 && (mailbox_get_private_flags_mask(box) & MAIL_SEEN) != 0) { /* can't get UNSEEN from list index, since each user has @@ -193,7 +199,7 @@ index_list_get_cached_status(struct mailbox *box, return 0; } - if ((ret = index_list_open_view(box, &view, &seq)) <= 0) + if ((ret = index_list_open_view(box, TRUE, &view, &seq)) <= 0) return ret; ret = mailbox_list_index_status(box->list, view, seq, items, @@ -230,7 +236,7 @@ index_list_get_cached_guid(struct mailbox *box, guid_128_t guid_r) return 0; } - if ((ret = index_list_open_view(box, &view, &seq)) <= 0) + if ((ret = index_list_open_view(box, FALSE, &view, &seq)) <= 0) return ret; ret = mailbox_list_index_status(box->list, view, seq, 0, @@ -252,7 +258,7 @@ static int index_list_get_cached_vsize(struct mailbox *box, uoff_t *vsize_r) i_assert(!ilist->syncing); - if ((ret = index_list_open_view(box, &view, &seq)) <= 0) + if ((ret = index_list_open_view(box, TRUE, &view, &seq)) <= 0) return ret; ret = mailbox_list_index_status(box->list, view, seq, @@ -283,7 +289,7 @@ index_list_get_cached_first_saved(struct mailbox *box, memset(first_saved_r, 0, sizeof(*first_saved_r)); - if ((ret = index_list_open_view(box, &view, &seq)) <= 0) + if ((ret = index_list_open_view(box, TRUE, &view, &seq)) <= 0) return ret; mail_index_lookup_ext(view, seq, ilist->first_saved_ext_id, @@ -700,7 +706,7 @@ void mailbox_list_index_update_mailbox_index(struct mailbox *box, int ret; memset(&changes, 0, sizeof(changes)); - if ((ret = index_list_open_view(box, &list_view, &changes.seq)) <= 0) + if ((ret = index_list_open_view(box, TRUE, &list_view, &changes.seq)) <= 0) return; (void)mailbox_list_index_status(box->list, list_view, changes.seq, @@ -786,8 +792,11 @@ void mailbox_list_index_status_set_info_flags(struct mailbox *box, uint32_t uid, /* our in-memory tree is out of sync */ ret = 1; } else T_BEGIN { + /* kludge: avoid breaking API for v2.2.x. Fixed in v2.3.x. */ + box->list_index_has_changed_quick = TRUE; ret = box->v.list_index_has_changed == NULL ? 0 : box->v.list_index_has_changed(box, view, seq); + box->list_index_has_changed_quick = FALSE; } T_END; if (ret != 0) { diff --git a/src/lib-storage/list/mailbox-list-iter.c b/src/lib-storage/list/mailbox-list-iter.c index bb8de56f5df..2c1381b834c 100644 --- a/src/lib-storage/list/mailbox-list-iter.c +++ b/src/lib-storage/list/mailbox-list-iter.c @@ -354,33 +354,38 @@ mailbox_list_ns_prefix_match(struct ns_list_iterate_context *ctx, return ret; } -static bool +static int ns_prefix_is_visible(struct ns_list_iterate_context *ctx, struct mail_namespace *ns) { + int ret; + if ((ns->flags & NAMESPACE_FLAG_LIST_PREFIX) != 0) - return TRUE; + return 1; if ((ns->flags & NAMESPACE_FLAG_LIST_CHILDREN) != 0) { - if (mailbox_list_match_anything(ctx, ns, ns->prefix)) - return TRUE; + if ((ret = mailbox_list_match_anything(ctx, ns, ns->prefix)) != 0) + return ret; } - return FALSE; + return 0; } -static bool +static int ns_prefix_has_visible_child_namespace(struct ns_list_iterate_context *ctx, const char *prefix) { struct mail_namespace *ns; unsigned int prefix_len = strlen(prefix); + int ret; for (ns = ctx->namespaces; ns != NULL; ns = ns->next) { if (ns->prefix_len > prefix_len && - strncmp(ns->prefix, prefix, prefix_len) == 0 && - ns_prefix_is_visible(ctx, ns)) - return TRUE; + strncmp(ns->prefix, prefix, prefix_len) == 0) { + ret = ns_prefix_is_visible(ctx, ns); + if (ret != 0) + return ret; + } } - return FALSE; + return 0; } static bool @@ -410,8 +415,8 @@ mailbox_list_match_anything(struct ns_list_iterate_context *ctx, const char *pattern; int ret; - if (ns_prefix_has_visible_child_namespace(ctx, prefix)) - return 1; + if ((ret = ns_prefix_has_visible_child_namespace(ctx, prefix)) != 0) + return ret; pattern = t_strconcat(prefix, "%", NULL); list_iter = mailbox_list_iter_init(ns->list, pattern, list_flags); diff --git a/src/lib-storage/mail-autoexpunge.c b/src/lib-storage/mail-autoexpunge.c index 840c184ba4a..9956f866b6c 100644 --- a/src/lib-storage/mail-autoexpunge.c +++ b/src/lib-storage/mail-autoexpunge.c @@ -8,32 +8,47 @@ #include "mail-user.h" #include "mail-autoexpunge.h" -static int mailbox_autoexpunge(struct mailbox *box, time_t expire_time) +static int +mailbox_autoexpunge(struct mailbox *box, unsigned int interval_time, + unsigned int max_mails) { struct mailbox_transaction_context *t; struct mail *mail; struct mailbox_metadata metadata; const struct mail_index_header *hdr; + struct mailbox_status status; uint32_t seq; - time_t timestamp; + time_t timestamp, expire_time; int ret = 0; + if ((unsigned int)ioloop_time < interval_time) + expire_time = 0; + else + expire_time = ioloop_time - interval_time; + /* first try to check quickly from mailbox list index if we should bother opening this mailbox. */ - if (mailbox_get_metadata(box, MAILBOX_METADATA_FIRST_SAVE_DATE, - &metadata) == 0) { - if (metadata.first_save_date == (time_t)-1 || - metadata.first_save_date > expire_time) - return 0; - } - - if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST) < 0) { + if (mailbox_get_status(box, STATUS_MESSAGES, &status) < 0) { if (mailbox_get_last_mail_error(box) == MAIL_ERROR_NOTFOUND) { /* autocreated mailbox doesn't exist yet */ return 0; } return -1; } + if (interval_time == 0 && status.messages <= max_mails) + return 0; + + if (max_mails == 0 || status.messages <= max_mails) { + if (mailbox_get_metadata(box, MAILBOX_METADATA_FIRST_SAVE_DATE, + &metadata) < 0) + return -1; + if (metadata.first_save_date == (time_t)-1 || + metadata.first_save_date > expire_time) + return 0; + } + + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST) < 0) + return -1; t = mailbox_transaction_begin(box, 0); mail = mail_alloc(t, 0, NULL); @@ -41,7 +56,15 @@ static int mailbox_autoexpunge(struct mailbox *box, time_t expire_time) hdr = mail_index_get_header(box->view); for (seq = 1; seq <= hdr->messages_count; seq++) { mail_set_seq(mail, seq); - if (mail_get_save_date(mail, ×tamp) == 0) { + if (max_mails > 0 && hdr->messages_count - seq + 1 > max_mails) { + /* max_mails is still being reached -> expunge. + don't even check saved-dates before we're + below max_mails. */ + mail_expunge(mail); + } else if (interval_time == 0) { + /* only max_mails is used. nothing further to do. */ + break; + } else if (mail_get_save_date(mail, ×tamp) == 0) { if (timestamp > expire_time) break; mail_expunge(mail); @@ -61,18 +84,16 @@ static int mailbox_autoexpunge(struct mailbox *box, time_t expire_time) static void mailbox_autoexpunge_set(struct mail_namespace *ns, const char *vname, - unsigned int autoexpunge) + unsigned int autoexpunge, + unsigned int autoexpunge_max_mails) { struct mailbox *box; - time_t expire_time; - - expire_time = ioloop_time - autoexpunge; /* autoexpunge is configured by admin, so we can safely ignore any ACLs the user might normally have against expunging in the mailbox. */ box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_IGNORE_ACLS); - if (mailbox_autoexpunge(box, expire_time) < 0) { + if (mailbox_autoexpunge(box, autoexpunge, autoexpunge_max_mails) < 0) { i_error("Failed to autoexpunge mailbox '%s': %s", mailbox_get_vname(box), mailbox_get_last_error(box, NULL)); @@ -92,7 +113,8 @@ mailbox_autoexpunge_wildcards(struct mail_namespace *ns, MAILBOX_LIST_ITER_SKIP_ALIASES | MAILBOX_LIST_ITER_RETURN_NO_FLAGS); while ((info = mailbox_list_iter_next(iter)) != NULL) T_BEGIN { - mailbox_autoexpunge_set(ns, info->vname, set->autoexpunge); + mailbox_autoexpunge_set(ns, info->vname, set->autoexpunge, + set->autoexpunge_max_mails); } T_END; if (mailbox_list_iter_deinit(&iter) < 0) { i_error("Failed to iterate autoexpunge mailboxes '%s%s': %s", @@ -110,8 +132,8 @@ static void mail_namespace_autoexpunge(struct mail_namespace *ns) return; array_foreach(&ns->set->mailboxes, box_set) { - if ((*box_set)->autoexpunge == 0 || - (unsigned int)ioloop_time < (*box_set)->autoexpunge) + if ((*box_set)->autoexpunge == 0 && + (*box_set)->autoexpunge_max_mails == 0) continue; if (strpbrk((*box_set)->name, "*?") != NULL) @@ -122,7 +144,8 @@ static void mail_namespace_autoexpunge(struct mail_namespace *ns) vname = t_strndup(ns->prefix, ns->prefix_len - 1); else vname = t_strconcat(ns->prefix, (*box_set)->name, NULL); - mailbox_autoexpunge_set(ns, vname, (*box_set)->autoexpunge); + mailbox_autoexpunge_set(ns, vname, (*box_set)->autoexpunge, + (*box_set)->autoexpunge_max_mails); } } } diff --git a/src/lib-storage/mail-search.c b/src/lib-storage/mail-search.c index 0deaf397e51..3c7cd4801cd 100644 --- a/src/lib-storage/mail-search.c +++ b/src/lib-storage/mail-search.c @@ -105,6 +105,7 @@ void mail_search_arg_init(struct mail_search_args *args, thread_args->pool = args->pool; thread_args->args = arg->value.subargs; thread_args->simplified = TRUE; + thread_args->init_refcount = 1; /* simplification should have unnested all inthreads, so we'll assume that have_inthreads=FALSE */ @@ -609,7 +610,7 @@ bool mail_search_arg_one_equals(const struct mail_search_arg *arg1, case SEARCH_FLAGS: return arg1->value.flags == arg2->value.flags; case SEARCH_KEYWORDS: - return strcasecmp(arg1->value.str, arg2->value.str); + return strcasecmp(arg1->value.str, arg2->value.str) == 0; case SEARCH_BEFORE: case SEARCH_ON: diff --git a/src/lib-storage/mail-search.h b/src/lib-storage/mail-search.h index d8447faebb6..e0a9fd0aa2d 100644 --- a/src/lib-storage/mail-search.h +++ b/src/lib-storage/mail-search.h @@ -101,6 +101,7 @@ struct mail_search_arg { unsigned int match_always:1; /* result = 1 always */ unsigned int nonmatch_always:1; /* result = 0 always */ unsigned int fuzzy:1; /* use fuzzy matching for this arg */ + unsigned int no_fts:1; /* do NOT call FTS */ int result; /* -1 = unknown, 0 = unmatched, 1 = matched */ }; diff --git a/src/lib-storage/mail-storage-private.h b/src/lib-storage/mail-storage-private.h index d1f1a8250c8..38946336472 100644 --- a/src/lib-storage/mail-storage-private.h +++ b/src/lib-storage/mail-storage-private.h @@ -78,7 +78,10 @@ enum mail_storage_class_flags { MAIL_STORAGE_CLASS_FLAG_BINARY_DATA = 0x100, /* Message GUIDs can only be 128bit (always set mailbox_status.have_only_guid128) */ - MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUID128 = 0x200 + MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUID128 = 0x200, + /* Storage deletes all files internally - mailbox list's + delete_mailbox() shouldn't delete anything itself. */ + MAIL_STORAGE_CLASS_FLAG_NO_LIST_DELETES = 0x400 }; struct mail_binary_cache { @@ -147,6 +150,24 @@ struct mail_attachment_part { const char *content_type, *content_disposition; }; +struct virtual_mailbox_vfuncs { + /* convert backend UIDs to virtual UIDs. if some backend UID doesn't + exist in mailbox, it's simply ignored */ + void (*get_virtual_uids)(struct mailbox *box, + struct mailbox *backend_mailbox, + const ARRAY_TYPE(seq_range) *backend_uids, + ARRAY_TYPE(seq_range) *virtual_uids_r); + /* like get_virtual_uids(), but if a backend UID doesn't exist, + convert it to 0. */ + void (*get_virtual_uid_map)(struct mailbox *box, + struct mailbox *backend_mailbox, + const ARRAY_TYPE(seq_range) *backend_uids, + ARRAY_TYPE(uint32_t) *virtual_uids_r); + void (*get_virtual_backend_boxes)(struct mailbox *box, + ARRAY_TYPE(mailboxes) *mailboxes, + bool only_with_msgs); +}; + struct mailbox_vfuncs { bool (*is_readonly)(struct mailbox *box); @@ -271,6 +292,10 @@ struct mailbox_index_vsize { uint32_t message_count; }; +struct mailbox_index_pop3_uidl { + uint32_t max_uid_with_pop3_uidl; +}; + struct mailbox_index_first_saved { uint32_t uid; uint32_t timestamp; @@ -283,7 +308,9 @@ struct mailbox { struct mail_storage *storage; struct mailbox_list *list; - struct mailbox_vfuncs v, *vlast; + struct mailbox_vfuncs v, *vlast; + /* virtual mailboxes: */ + const struct virtual_mailbox_vfuncs *virtual_vfuncs; /* private: */ pool_t pool, metadata_pool; /* Linked list of all mailboxes in this storage */ @@ -323,6 +350,7 @@ struct mailbox { enum mailbox_feature enabled_features; struct mail_msgpart_partial_cache partial_cache; uint32_t vsize_hdr_ext_id; + uint32_t pop3_uidl_hdr_ext_id; /* MAIL_RECENT flags handling */ ARRAY_TYPE(seq_range) recent_flags; @@ -382,6 +410,8 @@ struct mailbox { unsigned int update_first_saved:1; /* mailbox_verify_create_name() only checks for mailbox_verify_name() */ unsigned int skip_create_name_restrictions:1; + /* v2.2.x API kludge: quick-parameter to list_index_has_changed() */ + unsigned int list_index_has_changed_quick:1; }; struct mail_vfuncs { @@ -528,6 +558,9 @@ struct mailbox_transaction_context { struct mail_transaction_commit_changes *changes; ARRAY(union mailbox_transaction_module_context *) module_contexts; + uint32_t prev_pop3_uidl_tracking_seq; + uint32_t highest_pop3_uidl_uid; + struct mail_save_context *save_ctx; /* number of mails saved/copied within this transaction. */ unsigned int save_count; diff --git a/src/lib-storage/mail-storage-service.c b/src/lib-storage/mail-storage-service.c index babdab9804d..b347e7e157e 100644 --- a/src/lib-storage/mail-storage-service.c +++ b/src/lib-storage/mail-storage-service.c @@ -82,6 +82,8 @@ struct mail_storage_service_user { const struct setting_parser_info *user_info; struct setting_parser_context *set_parser; + unsigned int session_id_counter; + unsigned int anonymous:1; unsigned int admin:1; }; @@ -658,8 +660,15 @@ mail_storage_service_init_post(struct mail_storage_service_ctx *ctx, mail_user->admin = user->admin; mail_user->auth_token = p_strdup(mail_user->pool, user->auth_token); mail_user->auth_user = p_strdup(mail_user->pool, user->auth_user); - mail_user->session_id = - p_strdup(mail_user->pool, user->input.session_id); + if (user->session_id_counter++ == 0) { + mail_user->session_id = + p_strdup(mail_user->pool, user->input.session_id); + } else { + mail_user->session_id = + p_strdup_printf(mail_user->pool, "%s:%u", + user->input.session_id, + user->session_id_counter); + } mail_user->userdb_fields = user->input.userdb_fields == NULL ? NULL : p_strarray_dup(mail_user->pool, user->input.userdb_fields); mail_user->autoexpunge_enabled = @@ -1520,6 +1529,12 @@ mail_storage_service_all_iter_deinit(struct mail_storage_service_ctx *ctx) } void mail_storage_service_all_init(struct mail_storage_service_ctx *ctx) +{ + mail_storage_service_all_init_mask(ctx, ""); +} + +void mail_storage_service_all_init_mask(struct mail_storage_service_ctx *ctx, + const char *user_mask_hint) { enum auth_master_flags flags = 0; @@ -1533,7 +1548,8 @@ void mail_storage_service_all_init(struct mail_storage_service_ctx *ctx) flags |= AUTH_MASTER_FLAG_DEBUG; ctx->iter_conn = auth_master_init(auth_master_get_socket_path(ctx->conn), flags); - ctx->auth_list = auth_master_user_list_init(ctx->iter_conn, "", NULL); + ctx->auth_list = auth_master_user_list_init(ctx->iter_conn, + user_mask_hint, NULL); } int mail_storage_service_all_next(struct mail_storage_service_ctx *ctx, diff --git a/src/lib-storage/mail-storage-service.h b/src/lib-storage/mail-storage-service.h index 65b633194d1..603aeaccb7f 100644 --- a/src/lib-storage/mail-storage-service.h +++ b/src/lib-storage/mail-storage-service.h @@ -111,6 +111,11 @@ int mail_storage_service_lookup_next(struct mail_storage_service_ctx *ctx, void mail_storage_service_user_free(struct mail_storage_service_user **user); /* Initialize iterating through all users. */ void mail_storage_service_all_init(struct mail_storage_service_ctx *ctx); +/* Same as mail_storage_service_all_init(), but give a user mask hint to the + userdb iteration lookup. This itself isn't yet guaranteed to filter out any + usernames. */ +void mail_storage_service_all_init_mask(struct mail_storage_service_ctx *ctx, + const char *user_mask_hint); /* Iterate through all usernames. Returns 1 if username was returned, 0 if there are no more users, -1 if error. */ int mail_storage_service_all_next(struct mail_storage_service_ctx *ctx, diff --git a/src/lib-storage/mail-storage-settings.c b/src/lib-storage/mail-storage-settings.c index e2233bfb7c5..d1998fa0566 100644 --- a/src/lib-storage/mail-storage-settings.c +++ b/src/lib-storage/mail-storage-settings.c @@ -127,6 +127,7 @@ static const struct setting_define mailbox_setting_defines[] = { DEF(SET_STR, driver), DEF(SET_STR, comment), DEF(SET_TIME, autoexpunge), + DEF(SET_UINT, autoexpunge_max_mails), SETTING_DEFINE_LIST_END }; @@ -139,7 +140,8 @@ const struct mailbox_settings mailbox_default_settings = { .special_use = "", .driver = "", .comment = "", - .autoexpunge = 0 + .autoexpunge = 0, + .autoexpunge_max_mails = 0 }; const struct setting_parser_info mailbox_setting_parser_info = { diff --git a/src/lib-storage/mail-storage-settings.h b/src/lib-storage/mail-storage-settings.h index 44196a2147d..152872fc6b0 100644 --- a/src/lib-storage/mail-storage-settings.h +++ b/src/lib-storage/mail-storage-settings.h @@ -82,6 +82,7 @@ struct mailbox_settings { const char *driver; const char *comment; unsigned int autoexpunge; + unsigned int autoexpunge_max_mails; }; struct mail_user_settings { diff --git a/src/lib-storage/mail-storage.c b/src/lib-storage/mail-storage.c index 331b867d82f..5c71eb68543 100644 --- a/src/lib-storage/mail-storage.c +++ b/src/lib-storage/mail-storage.c @@ -360,6 +360,8 @@ int mail_storage_create_full(struct mail_namespace *ns, const char *driver, list_flags |= MAILBOX_LIST_FLAG_MAILBOX_FILES; if ((storage_class->class_flags & MAIL_STORAGE_CLASS_FLAG_NO_ROOT) != 0) list_flags |= MAILBOX_LIST_FLAG_NO_MAIL_FILES; + if ((storage_class->class_flags & MAIL_STORAGE_CLASS_FLAG_NO_LIST_DELETES) != 0) + list_flags |= MAILBOX_LIST_FLAG_NO_DELETES; if (mailbox_list_create(list_set.layout, ns, &list_set, list_flags, &list, error_r) < 0) { *error_r = t_strdup_printf("Mailbox list driver %s: %s", @@ -1519,6 +1521,12 @@ int mailbox_set_subscribed(struct mailbox *box, bool set) { if (mailbox_verify_name(box) < 0) return -1; + if (mailbox_list_iter_subscriptions_refresh(box->list) < 0) { + mail_storage_copy_list_error(box->storage, box->list); + return -1; + } + if (mailbox_is_subscribed(box) == set) + return 0; return box->v.set_subscribed(box, set); } diff --git a/src/lib-storage/mail-storage.h b/src/lib-storage/mail-storage.h index 4c99849e50b..64804b4f334 100644 --- a/src/lib-storage/mail-storage.h +++ b/src/lib-storage/mail-storage.h @@ -49,7 +49,13 @@ enum mailbox_flags { /* Force opening mailbox and ignoring any ACLs */ MAILBOX_FLAG_IGNORE_ACLS = 0x100, /* Open mailbox even if it's already marked as deleted */ - MAILBOX_FLAG_OPEN_DELETED = 0x200 + MAILBOX_FLAG_OPEN_DELETED = 0x200, + /* Mailbox is opened for deletion, which should be performed as + efficiently as possible, even allowing the mailbox state to become + inconsistent. For example this disables lazy_expunge plugin and + quota updates (possibly resulting in broken quota). and This is + useful for example when deleting entire user accounts. */ + MAILBOX_FLAG_DELETE_UNSAFE = 0x400 }; enum mailbox_feature { diff --git a/src/lib-storage/mailbox-guid-cache.c b/src/lib-storage/mailbox-guid-cache.c index f6ac6233df7..75488a18fb4 100644 --- a/src/lib-storage/mailbox-guid-cache.c +++ b/src/lib-storage/mailbox-guid-cache.c @@ -73,6 +73,11 @@ void mailbox_guid_cache_refresh(struct mailbox_list *list) i_error("Couldn't get mailbox %s GUID: %s", info->vname, mailbox_get_last_error(box, NULL)); list->guid_cache_errors = TRUE; + } else if ((rec = hash_table_lookup(list->guid_cache, + (const uint8_t *)metadata.guid)) != NULL) { + i_warning("Mailbox %s has duplicate GUID with %s: %s", + info->vname, rec->vname, + guid_128_to_string(metadata.guid)); } else { rec = p_new(list->guid_cache_pool, struct mailbox_guid_cache_rec, 1); diff --git a/src/lib-storage/mailbox-list.c b/src/lib-storage/mailbox-list.c index 001667a1e9c..72dc53087a0 100644 --- a/src/lib-storage/mailbox-list.c +++ b/src/lib-storage/mailbox-list.c @@ -866,6 +866,10 @@ void mailbox_list_get_default_storage(struct mailbox_list *list, char mailbox_list_get_hierarchy_sep(struct mailbox_list *list) { + /* the current API doesn't allow returning an error, so imap code + looks at the list's last error. make sure the error is cleared + so the error-check doesn't return something irrelevant */ + mailbox_list_clear_error(list); return list->v.get_hierarchy_sep(list); } diff --git a/src/lib-storage/mailbox-list.h b/src/lib-storage/mailbox-list.h index 8f5998c3ebe..ae953837bcb 100644 --- a/src/lib-storage/mailbox-list.h +++ b/src/lib-storage/mailbox-list.h @@ -34,7 +34,9 @@ enum mailbox_list_flags { mailbox list to it. */ MAILBOX_LIST_FLAG_SECONDARY = 0x02, /* There are no mail files, only index and/or control files. */ - MAILBOX_LIST_FLAG_NO_MAIL_FILES = 0x04 + MAILBOX_LIST_FLAG_NO_MAIL_FILES = 0x04, + /* LAYOUT=index: Don't delete any files in delete_mailbox(). */ + MAILBOX_LIST_FLAG_NO_DELETES = 0x08 }; enum mailbox_info_flags { diff --git a/src/lib-test/test-common.c b/src/lib-test/test-common.c index aff3c230da8..270e0effff4 100644 --- a/src/lib-test/test-common.c +++ b/src/lib-test/test-common.c @@ -38,8 +38,10 @@ static ssize_t test_read(struct istream_private *stream) i_assert(stream->skip <= stream->pos); - if (stream->pos - stream->skip >= tstream->istream.max_buffer_size) + if (stream->pos - stream->skip >= tstream->istream.max_buffer_size) { + i_assert(stream->skip != stream->pos); return -2; + } if (tstream->max_pos < stream->pos) { /* we seeked past the end of file. */ @@ -177,6 +179,15 @@ void test_assert_failed_idx(const char *code, const char *file, unsigned int lin test_success = FALSE; } +void test_assert_failed_strcmp(const char *code, const char *file, unsigned int line, + const char * src, const char * dst) +{ + printf("%s: Assert(#%u) failed: %s\n", file, line, code); + printf(" \"%s\" != \"%s\"\n", src, dst); + fflush(stdout); + test_success = FALSE; +} + static void test_dump_rand_state(void) { diff --git a/src/lib-test/test-common.h b/src/lib-test/test-common.h index cbb02100738..fc62afb52da 100644 --- a/src/lib-test/test-common.h +++ b/src/lib-test/test-common.h @@ -17,8 +17,17 @@ void test_begin(const char *name); #define test_assert_idx(code, i) STMT_START { \ if (!(code)) test_assert_failed_idx(#code, __FILE__, __LINE__, i); \ } STMT_END +/* Additional parameters are s1 (source) and s2 (destination) string + * in strcmp(). + */ +#define test_assert_strcmp(s1, s2) STMT_START { \ + if ((strcmp(s1,s2) != 0)) test_assert_failed_strcmp("strcmp(" #s1 "," #s2 ")", __FILE__, __LINE__, s1, s2); \ + } STMT_END + void test_assert_failed(const char *code, const char *file, unsigned int line); void test_assert_failed_idx(const char *code, const char *file, unsigned int line, long long i); +void test_assert_failed_strcmp(const char *code, const char *file, unsigned int line, + const char * src, const char * dst); bool test_has_failed(void); /* If you're testing nasty cases which you want to warn, surround the noisy op with these */ void test_expect_errors(unsigned int expected); diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am index d5705e90587..7845ffe5f64 100644 --- a/src/lib/Makefile.am +++ b/src/lib/Makefile.am @@ -114,6 +114,7 @@ liblib_la_SOURCES = \ ostream-hash.c \ ostream-null.c \ ostream-rawlog.c \ + pkcs5.c \ primes.c \ printf-format-fix.c \ process-title.c \ @@ -247,6 +248,7 @@ headers = \ ostream-private.h \ ostream-null.h \ ostream-rawlog.h \ + pkcs5.h \ primes.h \ printf-format-fix.h \ process-title.h \ @@ -328,6 +330,7 @@ test_lib_SOURCES = \ test-json-tree.c \ test-llist.c \ test-mempool-alloconly.c \ + test-pkcs5.c \ test-net.c \ test-numpack.c \ test-ostream-escaped.c \ diff --git a/src/lib/aqueue.c b/src/lib/aqueue.c index 26388bd68ee..efddd10ed95 100644 --- a/src/lib/aqueue.c +++ b/src/lib/aqueue.c @@ -10,7 +10,7 @@ struct aqueue *aqueue_init(struct array *array) aqueue = i_new(struct aqueue, 1); aqueue->arr = array; - aqueue->area_size = buffer_get_size(aqueue->arr->buffer) / + aqueue->area_size = buffer_get_writable_size(aqueue->arr->buffer) / aqueue->arr->element_size; i_assert(aqueue->area_size > 0); return aqueue; @@ -32,7 +32,7 @@ static void aqueue_grow(struct aqueue *aqueue) orig_area_size = aqueue->area_size; (void)array_append_space_i(aqueue->arr); - aqueue->area_size = buffer_get_size(aqueue->arr->buffer) / + aqueue->area_size = buffer_get_writable_size(aqueue->arr->buffer) / aqueue->arr->element_size; i_assert(orig_area_size < aqueue->area_size); diff --git a/src/lib/buffer.c b/src/lib/buffer.c index d7430c0fbff..aef163595f4 100644 --- a/src/lib/buffer.c +++ b/src/lib/buffer.c @@ -30,7 +30,10 @@ static void buffer_alloc(struct real_buffer *buf, size_t size) i_assert(size > buf->alloc); - buf->w_buffer = p_realloc(buf->pool, buf->w_buffer, buf->alloc, size); + if (buf->w_buffer == NULL) + buf->w_buffer = p_malloc(buf->pool, size); + else + buf->w_buffer = p_realloc(buf->pool, buf->w_buffer, buf->alloc, size); buf->alloc = size; buf->r_buffer = buf->w_buffer; @@ -134,7 +137,10 @@ buffer_t *buffer_create_dynamic(pool_t pool, size_t init_size) buf = p_new(pool, struct real_buffer, 1); buf->pool = pool; buf->dynamic = TRUE; - buffer_alloc(buf, init_size); + /* buffer_alloc() reserves +1 for str_c() NIL, so add +1 here to + init_size so we can actually write that much to the buffer without + realloc */ + buffer_alloc(buf, init_size+1); return (buffer_t *)buf; } diff --git a/src/lib/connection.c b/src/lib/connection.c index f9fab8f798b..b73a8c32412 100644 --- a/src/lib/connection.c +++ b/src/lib/connection.c @@ -359,7 +359,26 @@ int connection_input_read(struct connection *conn) const char *connection_disconnect_reason(struct connection *conn) { - return io_stream_get_disconnect_reason(conn->input, conn->output); + switch (conn->disconnect_reason) { + case CONNECTION_DISCONNECT_DEINIT: + return "Deinitializing"; + case CONNECTION_DISCONNECT_CONNECT_TIMEOUT: { + unsigned int msecs = + conn->list->set.client_connect_timeout_msecs; + return t_strdup_printf("connect() timed out in %u.%03u secs", + msecs/1000, msecs%1000); + } + case CONNECTION_DISCONNECT_IDLE_TIMEOUT: + return "Idle timeout"; + case CONNECTION_DISCONNECT_CONN_CLOSED: + if (conn->input == NULL) + return t_strdup_printf("connect() failed: %m"); + /* fall through */ + case CONNECTION_DISCONNECT_NOT: + case CONNECTION_DISCONNECT_BUFFER_FULL: + return io_stream_get_disconnect_reason(conn->input, conn->output); + } + i_unreached(); } void connection_switch_ioloop(struct connection *conn) diff --git a/src/lib/file-create-locked.c b/src/lib/file-create-locked.c index 3ea4c66b703..905e8cdc01f 100644 --- a/src/lib/file-create-locked.c +++ b/src/lib/file-create-locked.c @@ -69,9 +69,11 @@ try_create_new(const char *path, const struct file_create_settings *set, /* just created by somebody else */ ret = 0; } else if (errno == ENOENT) { - /* our temp file was just deleted by somebody else, - retry creating it. */ - ret = 0; + /* nobody should be deleting the temp file unless the + entire directory is deleted. */ + *error_r = t_strdup_printf( + "Temporary file %s was unexpectedly deleted", + str_c(temp_path)); } else { *error_r = t_strdup_printf("link(%s, %s) failed: %m", str_c(temp_path), path); diff --git a/src/lib/guid.c b/src/lib/guid.c index 902c79658c5..86d746412e8 100644 --- a/src/lib/guid.c +++ b/src/lib/guid.c @@ -1,6 +1,7 @@ /* Copyright (c) 2011-2016 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "ioloop.h" #include "buffer.h" #include "sha1.h" #include "hash.h" @@ -35,7 +36,7 @@ const char *guid_generate(void) } void guid_128_host_hash_get(const char *host, - unsigned char hash_r[GUID_128_HOST_HASH_SIZE]) + unsigned char hash_r[STATIC_ARRAY GUID_128_HOST_HASH_SIZE]) { unsigned char full_hash[SHA1_RESULTLEN]; @@ -65,7 +66,15 @@ void guid_128_generate(guid_128_t guid_r) guid_static[2] = (pid & 0x00ff0000) >> 16; guid_static[3] = (pid & 0xff000000) >> 24; guid_128_host_hash_get(my_hostdomain(), guid_static+4); - } else if ((uint32_t)ts.tv_nsec < (uint32_t)-1) { + } else if (ioloop_timeval.tv_sec > ts.tv_sec || + (ioloop_timeval.tv_sec == ts.tv_sec && + ioloop_timeval.tv_usec > ts.tv_nsec*1000)) { + /* use ioloop's time since we have it. it doesn't provide any + more uniqueness, but it allows finding out more reliably + when a GUID was created. */ + ts.tv_sec = ioloop_timeval.tv_sec; + ts.tv_nsec = ioloop_timeval.tv_usec*1000; + } else if ((uint32_t)ts.tv_nsec < 1000000000) { ts.tv_nsec++; } else { ts.tv_sec++; diff --git a/src/lib/guid.h b/src/lib/guid.h index 28fc4a11651..867e3d0cb20 100644 --- a/src/lib/guid.h +++ b/src/lib/guid.h @@ -37,6 +37,6 @@ int guid_128_cmp(const guid_128_t guid1, const guid_128_t guid2) ATTR_PURE; /* Return the hash of host used by guid_128_generate(). */ void guid_128_host_hash_get(const char *host, - unsigned char hash_r[GUID_128_HOST_HASH_SIZE]); + unsigned char hash_r[STATIC_ARRAY GUID_128_HOST_HASH_SIZE]); #endif diff --git a/src/lib/ioloop-epoll.c b/src/lib/ioloop-epoll.c index 1143abecb61..c2fe1a94867 100644 --- a/src/lib/ioloop-epoll.c +++ b/src/lib/ioloop-epoll.c @@ -105,7 +105,7 @@ void io_loop_handle_add(struct io_file *io) if (epoll_ctl(ctx->epfd, op, io->fd, &event) < 0) { if (errno == EPERM && op == EPOLL_CTL_ADD) { - i_fatal("epoll_ctl(add, %d) failed: %m " + i_panic("epoll_ctl(add, %d) failed: %m " "(fd doesn't support epoll%s)", io->fd, io->fd != STDIN_FILENO ? "" : " - instead of 'v_offset <= in_size); if (outstream->dupstream == NULL) { outstream->dupstream = instream; @@ -185,8 +186,10 @@ static off_t o_stream_temp_send_istream(struct ostream_private *_outstream, if ((outstream->flags & IOSTREAM_TEMP_FLAG_TRY_FD_DUP) != 0) { orig_offset = outstream->dupstream_offset; - if ((ret = o_stream_temp_dup_istream(outstream, instream)) > 0) + if ((ret = o_stream_temp_dup_istream(outstream, instream)) > 0) { + i_assert(outstream->dupstream_offset >= orig_offset); return outstream->dupstream_offset - orig_offset; + } if (ret < 0) return -1; outstream->flags &= ~IOSTREAM_TEMP_FLAG_TRY_FD_DUP; diff --git a/src/lib/istream-chain.c b/src/lib/istream-chain.c index 27fd43778a9..6880f8d00e2 100644 --- a/src/lib/istream-chain.c +++ b/src/lib/istream-chain.c @@ -24,6 +24,7 @@ struct chain_istream { struct istream_private istream; size_t prev_stream_left, prev_skip; + bool have_explicit_max_buffer_size; struct istream_chain chain; }; @@ -45,12 +46,16 @@ i_stream_chain_append_internal(struct istream_chain *chain, i_stream_ref(stream); if (chain->head == NULL && stream != NULL) { - if (chain->stream->istream.max_buffer_size == 0) { - chain->stream->istream.max_buffer_size = - stream->real_stream->max_buffer_size; - } else { + struct chain_istream *cstream = (struct chain_istream *)chain->stream; + + if (cstream->have_explicit_max_buffer_size) { i_stream_set_max_buffer_size(stream, chain->stream->istream.max_buffer_size); + } else { + size_t max_size = i_stream_get_max_buffer_size(stream); + + if (cstream->istream.max_buffer_size < max_size) + cstream->istream.max_buffer_size = max_size; } } DLLIST2_APPEND(&chain->head, &chain->tail, link); @@ -77,6 +82,7 @@ i_stream_chain_set_max_buffer_size(struct iostream_private *stream, struct chain_istream *cstream = (struct chain_istream *)stream; struct istream_chain_link *link = cstream->chain.head; + cstream->have_explicit_max_buffer_size = TRUE; cstream->istream.max_buffer_size = max_size; while (link != NULL) { if (link->stream != NULL) @@ -106,7 +112,7 @@ static void i_stream_chain_read_next(struct chain_istream *cstream) struct istream_chain_link *link = cstream->chain.head; struct istream *prev_input; const unsigned char *data; - size_t data_size, size, cur_data_pos; + size_t data_size, cur_data_pos; i_assert(link != NULL && link->stream != NULL); i_assert(link->stream->eof); @@ -137,17 +143,12 @@ static void i_stream_chain_read_next(struct chain_istream *cstream) cstream->prev_stream_left = 0; } - /* we already verified that the data size is less than the - maximum buffer size */ if (data_size > 0) { - if (!i_stream_try_alloc(&cstream->istream, data_size, &size)) - i_unreached(); - i_assert(size >= data_size); + memcpy(i_stream_alloc(&cstream->istream, data_size), + data, data_size); + cstream->istream.pos += data_size; + cstream->prev_stream_left += data_size; } - memcpy(cstream->istream.w_buffer + cstream->istream.pos, - data, data_size); - cstream->istream.pos += data_size; - cstream->prev_stream_left += data_size; i_stream_skip(prev_input, i_stream_get_data_size(prev_input)); i_stream_unref(&prev_input); @@ -190,7 +191,7 @@ static ssize_t i_stream_chain_read(struct istream_private *stream) struct chain_istream *cstream = (struct chain_istream *)stream; struct istream_chain_link *link = cstream->chain.head; const unsigned char *data; - size_t size, data_size, cur_data_pos, new_pos; + size_t data_size, cur_data_pos, new_pos; size_t new_bytes_count; ssize_t ret; @@ -251,16 +252,9 @@ static ssize_t i_stream_chain_read(struct istream_private *stream) new data with it. */ i_assert(data_size > cur_data_pos); new_bytes_count = data_size - cur_data_pos; - if (!i_stream_try_alloc(stream, new_bytes_count, &size)) { - stream->buffer = stream->w_buffer; - return -2; - } - stream->buffer = stream->w_buffer; - - if (new_bytes_count > size) - new_bytes_count = size; - memcpy(stream->w_buffer + stream->pos, + memcpy(i_stream_alloc(stream, new_bytes_count), data + cur_data_pos, new_bytes_count); + stream->buffer = stream->w_buffer; new_pos = stream->pos + new_bytes_count; } @@ -295,7 +289,6 @@ struct istream *i_stream_create_chain(struct istream_chain **chain_r) cstream = i_new(struct chain_istream, 1); cstream->chain.stream = cstream; - cstream->istream.max_buffer_size = 256; cstream->istream.iostream.close = i_stream_chain_close; cstream->istream.iostream.destroy = i_stream_chain_destroy; diff --git a/src/lib/istream-concat.c b/src/lib/istream-concat.c index f490ec31717..a6937045746 100644 --- a/src/lib/istream-concat.c +++ b/src/lib/istream-concat.c @@ -23,7 +23,10 @@ static void i_stream_concat_close(struct iostream_private *stream, struct concat_istream *cstream = (struct concat_istream *)stream; unsigned int i; - (void)i_stream_concat_skip(cstream); + if (cstream->istream.istream.stream_errno == 0) { + /* get the parent streams to the wanted offset */ + (void)i_stream_concat_skip(cstream); + } if (close_parent) { for (i = 0; cstream->input[i] != NULL; i++) @@ -166,7 +169,7 @@ static ssize_t i_stream_concat_read(struct istream_private *stream) /* we either read something or we're at EOF */ last_stream = cstream->input[cstream->cur_idx+1] == NULL; if (ret == -1 && !last_stream) { - if (stream->pos - stream->skip >= stream->max_buffer_size) + if (stream->pos - stream->skip >= i_stream_get_max_buffer_size(&stream->istream)) return -2; i_stream_concat_read_next(cstream); @@ -317,7 +320,7 @@ struct istream *i_stream_create_concat(struct istream *input[]) /* if any of the streams isn't blocking or seekable, set ourself also nonblocking/nonseekable */ for (count = 0; input[count] != NULL; count++) { - size_t cur_max = input[count]->real_stream->max_buffer_size; + size_t cur_max = i_stream_get_max_buffer_size(input[count]); if (cur_max > max_buffer_size) max_buffer_size = cur_max; diff --git a/src/lib/istream-seekable.c b/src/lib/istream-seekable.c index a12581813c9..f6340be5f95 100644 --- a/src/lib/istream-seekable.c +++ b/src/lib/istream-seekable.c @@ -316,25 +316,24 @@ i_stream_seekable_stat(struct istream_private *stream, bool exact) return 0; } - if (sstream->membuf != NULL) { - /* we want to know the full size of the file, so read until - we're finished */ - old_offset = stream->istream.v_offset; - do { - i_stream_skip(&stream->istream, - stream->pos - stream->skip); - } while ((ret = i_stream_seekable_read(stream)) > 0); - - if (ret == 0) { - i_panic("i_stream_stat() used for non-blocking " - "seekable stream %s offset %"PRIuUOFF_T, - i_stream_get_name(sstream->cur_input), - sstream->cur_input->v_offset); - } - i_stream_skip(&stream->istream, stream->pos - stream->skip); - i_stream_seek(&stream->istream, old_offset); - unref_streams(sstream); + /* we want to know the full size of the file, so read until + we're finished */ + old_offset = stream->istream.v_offset; + do { + i_stream_skip(&stream->istream, + stream->pos - stream->skip); + } while ((ret = i_stream_seekable_read(stream)) > 0); + + if (ret == 0) { + i_panic("i_stream_stat() used for non-blocking " + "seekable stream %s offset %"PRIuUOFF_T, + i_stream_get_name(sstream->cur_input), + sstream->cur_input->v_offset); } + i_stream_skip(&stream->istream, stream->pos - stream->skip); + i_stream_seek(&stream->istream, old_offset); + unref_streams(sstream); + if (stream->istream.stream_errno != 0) return -1; diff --git a/src/lib/istream-timeout.c b/src/lib/istream-timeout.c index b3dd4719020..50b23ffc4ac 100644 --- a/src/lib/istream-timeout.c +++ b/src/lib/istream-timeout.c @@ -38,7 +38,7 @@ static void i_stream_timeout_switch_ioloop(struct istream_private *stream) static void i_stream_timeout(struct timeout_istream *tstream) { - unsigned int msecs; + unsigned int over_msecs; int diff; if (tstream->update_timestamp) { @@ -59,13 +59,14 @@ static void i_stream_timeout(struct timeout_istream *tstream) i_stream_timeout, tstream); return; } + over_msecs = diff - tstream->timeout_msecs; - msecs = tstream->timeout_msecs % 1000; io_stream_set_error(&tstream->istream.iostream, - "Read timeout in %u%s s after %"PRIuUOFF_T" bytes", - tstream->timeout_msecs/1000, - msecs == 0 ? "" : t_strdup_printf(".%u", msecs), - tstream->istream.istream.v_offset); + "Read timeout in %u.%03u s after %"PRIuUOFF_T" bytes%s", + diff/1000, diff%1000, + tstream->istream.istream.v_offset, + over_msecs < 1000 ? "" : t_strdup_printf( + " (requested timeout in %u ms)", tstream->timeout_msecs)); tstream->istream.istream.stream_errno = ETIMEDOUT; i_stream_set_input_pending(tstream->istream.parent, TRUE); diff --git a/src/lib/istream.c b/src/lib/istream.c index 2d70a230db7..aa95c1cb9d7 100644 --- a/src/lib/istream.c +++ b/src/lib/istream.c @@ -97,6 +97,11 @@ const char *i_stream_get_error(struct istream *stream) return strerror(stream->stream_errno); } +const char *i_stream_get_disconnect_reason(struct istream *stream) +{ + return io_stream_get_disconnect_reason(stream, NULL); +} + void i_stream_close(struct istream *stream) { i_stream_close_full(stream, TRUE); @@ -114,7 +119,14 @@ void i_stream_set_max_buffer_size(struct istream *stream, size_t max_size) size_t i_stream_get_max_buffer_size(struct istream *stream) { - return stream->real_stream->max_buffer_size; + size_t max_size = 0; + + do { + if (max_size < stream->real_stream->max_buffer_size) + max_size = stream->real_stream->max_buffer_size; + stream = stream->real_stream->parent; + } while (stream != NULL); + return max_size; } void i_stream_set_return_partial_line(struct istream *stream, bool set) @@ -584,9 +596,7 @@ void i_stream_compress(struct istream_private *stream) void i_stream_grow_buffer(struct istream_private *stream, size_t bytes) { - size_t old_size; - - i_assert(stream->max_buffer_size > 0); + size_t old_size, max_size; old_size = stream->buffer_size; @@ -596,8 +606,10 @@ void i_stream_grow_buffer(struct istream_private *stream, size_t bytes) else stream->buffer_size = nearest_power(stream->buffer_size); - if (stream->buffer_size > stream->max_buffer_size) - stream->buffer_size = stream->max_buffer_size; + max_size = i_stream_get_max_buffer_size(&stream->istream); + i_assert(max_size > 0); + if (stream->buffer_size > max_size) + stream->buffer_size = max_size; if (stream->buffer_size <= old_size) stream->buffer_size = old_size; @@ -617,8 +629,7 @@ bool i_stream_try_alloc(struct istream_private *stream, if (stream->skip > 0) { /* remove the unused bytes from beginning of buffer */ i_stream_compress(stream); - } else if (stream->max_buffer_size == 0 || - stream->buffer_size < stream->max_buffer_size) { + } else if (stream->buffer_size < i_stream_get_max_buffer_size(&stream->istream)) { /* buffer is full - grow it */ i_stream_grow_buffer(stream, I_STREAM_MIN_SIZE); } @@ -803,10 +814,8 @@ static int i_stream_default_get_size(struct istream_private *stream, bool exact, uoff_t *size_r) { - if (stream->stat(stream, exact) < 0) { - stream->istream.stream_errno = stream->parent->stream_errno; + if (stream->stat(stream, exact) < 0) return -1; - } if (stream->statbuf.st_size == -1) return 0; diff --git a/src/lib/istream.h b/src/lib/istream.h index 0c96e989671..34f95715d1d 100644 --- a/src/lib/istream.h +++ b/src/lib/istream.h @@ -77,6 +77,10 @@ int i_stream_get_fd(struct istream *stream); /* Returns error string for the last error. It also returns "EOF" in case there is no error, but eof is set. Otherwise it returns "". */ const char *i_stream_get_error(struct istream *stream); +/* Returns human-readable reason for why istream was disconnected. This can be + called to log the error when i_stream_read() returns -1. If there's an error + the output is identical to i_stream_get_error(). */ +const char *i_stream_get_disconnect_reason(struct istream *stream); /* Mark the stream and all of its parent streams closed. Any reads after this will return -1. The data already read can still be used. */ @@ -90,9 +94,13 @@ void i_stream_sync(struct istream *stream); unless it's called before reading anything. */ void i_stream_set_init_buffer_size(struct istream *stream, size_t size); /* Change the maximum size for stream's input buffer to grow. Useful only - for buffered streams (currently only file). */ + for buffered streams (currently only file). This changes also all the + parent streams' max buffer size. */ void i_stream_set_max_buffer_size(struct istream *stream, size_t max_size); -/* Returns the current max. buffer size. */ +/* Returns the current max. buffer size for the stream. This function also + goesthrough all of the parent streams and returns the highest seen max + buffer size. This is needed because some streams (e.g. istream-chain) change + their max buffer size dynamically. */ size_t i_stream_get_max_buffer_size(struct istream *stream); /* Enable/disable i_stream[_read]_next_line() returning the last line if it doesn't end with LF. */ @@ -161,6 +169,24 @@ unsigned char *i_stream_get_modifiable_data(struct istream *stream, input buffer is full. */ int i_stream_read_data(struct istream *stream, const unsigned char **data_r, size_t *size_r, size_t threshold); +/* Like i_stream_get_data(), but read more when needed. Returns 1 if at least + the wanted number of bytes are available, 0 if less, -1 if error or + EOF with no bytes read that weren't already in buffer, or -2 if stream's + input buffer is full. */ +static inline int +i_stream_read_bytes(struct istream *stream, const unsigned char **data_r, + size_t *size_r, size_t wanted) +{ + i_assert(wanted > 0); + return i_stream_read_data(stream, data_r, size_r, wanted - 1); +} +/* Short-hand for just requesting more data (i.e. even one byte) */ +static inline int +i_stream_read_more(struct istream *stream, const unsigned char **data_r, + size_t *size_r) +{ + return i_stream_read_bytes(stream, data_r, size_r, 1); +} /* Append external data to input stream. Returns TRUE if successful, FALSE if there is not enough space in the stream. */ diff --git a/src/lib/json-parser.c b/src/lib/json-parser.c index fe54a19cdf1..4e6afbf85c1 100644 --- a/src/lib/json-parser.c +++ b/src/lib/json-parser.c @@ -133,8 +133,9 @@ int json_parser_deinit(struct json_parser **_parser, const char **error_r) /* actual parser error */ *error_r = parser->error; } else if (parser->input->stream_errno != 0) { - *error_r = t_strdup_printf("read(%s) failed: %m", - i_stream_get_name(parser->input)); + *error_r = t_strdup_printf("read(%s) failed: %s", + i_stream_get_name(parser->input), + i_stream_get_error(parser->input)); } else if (parser->data == parser->end && !i_stream_have_bytes_left(parser->input) && parser->state != JSON_STATE_DONE) { diff --git a/src/lib/macros.h b/src/lib/macros.h index f8e4689c193..593ca49d44c 100644 --- a/src/lib/macros.h +++ b/src/lib/macros.h @@ -226,4 +226,9 @@ # define DOVECOT_PREREQ(maj, min) 0 #endif +#ifdef __cplusplus +# undef STATIC_ARRAY +# define STATIC_ARRAY +#endif + #endif diff --git a/src/lib/md4.c b/src/lib/md4.c index fade4d44512..de59b38c81f 100644 --- a/src/lib/md4.c +++ b/src/lib/md4.c @@ -208,7 +208,7 @@ void md4_update(struct md4_context *ctx, const void *data, size_t size) memcpy(ctx->buffer, data, size); } -void md4_final(struct md4_context *ctx, unsigned char result[MD4_RESULTLEN]) +void md4_final(struct md4_context *ctx, unsigned char result[STATIC_ARRAY MD4_RESULTLEN]) { /* @UNSAFE */ unsigned long used, free; @@ -261,7 +261,7 @@ void md4_final(struct md4_context *ctx, unsigned char result[MD4_RESULTLEN]) } void md4_get_digest(const void *data, size_t size, - unsigned char result[MD4_RESULTLEN]) + unsigned char result[STATIC_ARRAY MD4_RESULTLEN]) { struct md4_context ctx; diff --git a/src/lib/md4.h b/src/lib/md4.h index 771e08c850e..1530d0d0ff5 100644 --- a/src/lib/md4.h +++ b/src/lib/md4.h @@ -22,10 +22,11 @@ struct md4_context { void md4_init(struct md4_context *ctx); void md4_update(struct md4_context *ctx, const void *data, size_t size); -void md4_final(struct md4_context *ctx, unsigned char result[MD4_RESULTLEN]); +void md4_final(struct md4_context *ctx, + unsigned char result[STATIC_ARRAY MD4_RESULTLEN]); void md4_get_digest(const void *data, size_t size, - unsigned char result[MD4_RESULTLEN]); + unsigned char result[STATIC_ARRAY MD4_RESULTLEN]); extern const struct hash_method hash_method_md4; diff --git a/src/lib/md5.c b/src/lib/md5.c index 6957e054254..16264290c55 100644 --- a/src/lib/md5.c +++ b/src/lib/md5.c @@ -220,7 +220,7 @@ void md5_update(struct md5_context *ctx, const void *data, size_t size) memcpy(ctx->buffer, data, size); } -void md5_final(struct md5_context *ctx, unsigned char result[MD5_RESULTLEN]) +void md5_final(struct md5_context *ctx, unsigned char result[STATIC_ARRAY MD5_RESULTLEN]) { /* @UNSAFE */ unsigned long used, free; @@ -273,7 +273,7 @@ void md5_final(struct md5_context *ctx, unsigned char result[MD5_RESULTLEN]) } void md5_get_digest(const void *data, size_t size, - unsigned char result[MD5_RESULTLEN]) + unsigned char result[STATIC_ARRAY MD5_RESULTLEN]) { struct md5_context ctx; diff --git a/src/lib/md5.h b/src/lib/md5.h index 3da87fba375..682a6c5fe94 100644 --- a/src/lib/md5.h +++ b/src/lib/md5.h @@ -22,10 +22,11 @@ struct md5_context { void md5_init(struct md5_context *ctx); void md5_update(struct md5_context *ctx, const void *data, size_t size); -void md5_final(struct md5_context *ctx, unsigned char result[MD5_RESULTLEN]); +void md5_final(struct md5_context *ctx, + unsigned char result[STATIC_ARRAY MD5_RESULTLEN]); void md5_get_digest(const void *data, size_t size, - unsigned char result[MD5_RESULTLEN]); + unsigned char result[STATIC_ARRAY MD5_RESULTLEN]); extern const struct hash_method hash_method_md5; diff --git a/src/lib/mempool-system.c b/src/lib/mempool-system.c index ab83e22310c..2cc5d3ae6f6 100644 --- a/src/lib/mempool-system.c +++ b/src/lib/mempool-system.c @@ -119,6 +119,10 @@ static void *pool_system_realloc(pool_t pool ATTR_UNUSED, void *mem, if (unlikely(new_size == 0 || new_size > SSIZE_T_MAX)) i_panic("Trying to allocate %"PRIuSIZE_T" bytes", new_size); + if (mem == NULL) { + i_assert(old_size == 0); + return pool_system_malloc(pool, new_size); + } #if !defined(USE_GC) && defined(HAVE_MALLOC_USABLE_SIZE) i_assert(old_size == (size_t)-1 || mem == NULL || old_size <= malloc_usable_size(mem)); diff --git a/src/lib/net.c b/src/lib/net.c index 5223a544198..7450c1597a8 100644 --- a/src/lib/net.c +++ b/src/lib/net.c @@ -374,6 +374,30 @@ int net_set_cork(int fd ATTR_UNUSED, bool cork ATTR_UNUSED) #endif } +int net_set_send_buffer_size(int fd, size_t size) +{ + int opt; + + if (size > INT_MAX) { + errno = EINVAL; + return -1; + } + opt = (int)size; + return setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &opt, sizeof(opt)); +} + +int net_set_recv_buffer_size(int fd, size_t size) +{ + int opt; + + if (size > INT_MAX) { + errno = EINVAL; + return -1; + } + opt = (int)size; + return setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &opt, sizeof(opt)); +} + void net_get_ip_any4(struct ip_addr *ip) { ip->family = AF_INET; diff --git a/src/lib/net.h b/src/lib/net.h index 59d89a6ad76..0a037705c91 100644 --- a/src/lib/net.h +++ b/src/lib/net.h @@ -93,6 +93,10 @@ void net_set_nonblock(int fd, bool nonblock); Returns 0 if ok, -1 if failed. */ int net_set_cork(int fd, bool cork) ATTR_NOWARN_UNUSED_RESULT; +/* Set socket kernel buffer sizes */ +int net_set_send_buffer_size(int fd, size_t size); +int net_set_recv_buffer_size(int fd, size_t size); + /* Set IP to contain INADDR_ANY for IPv4 or IPv6. The IPv6 any address may include IPv4 depending on the system (Linux yes, BSD no). */ void net_get_ip_any4(struct ip_addr *ip); diff --git a/src/lib/ostream-escaped.c b/src/lib/ostream-escaped.c index c875d70c7ee..b3613ca32b4 100644 --- a/src/lib/ostream-escaped.c +++ b/src/lib/ostream-escaped.c @@ -40,7 +40,7 @@ static ssize_t o_stream_escaped_send_chunk(struct escaped_ostream *estream, const unsigned char *data, size_t len) { - size_t i, max_buffer_size, flush_pos; + size_t i, max_buffer_size; ssize_t ret; max_buffer_size = I_MIN(o_stream_get_max_buffer_size(estream->ostream.parent), @@ -50,24 +50,20 @@ o_stream_escaped_send_chunk(struct escaped_ostream *estream, max_buffer_size = IO_BLOCK_SIZE; } - flush_pos = str_len(estream->buf); for (i = 0; i < len; i++) { if (str_len(estream->buf) + 2 > max_buffer_size) { /* escaping takes at least two bytes */ - estream->ostream.ostream.offset += - str_len(estream->buf) - flush_pos; ret = o_stream_escaped_send_outbuf(estream); - if (ret < 0) + if (ret < 0) { + estream->ostream.ostream.offset += i; return ret; - flush_pos = str_len(estream->buf); + } if (ret == 0) break; } estream->format(estream->buf, data[i]); estream->flushed = FALSE; } - /* we'll return how many bytes of input we consumed, but ostream offset - contains how many bytes we actually wrote */ - estream->ostream.ostream.offset += str_len(estream->buf) - flush_pos; + estream->ostream.ostream.offset += i; return i; } diff --git a/src/lib/pkcs5.c b/src/lib/pkcs5.c new file mode 100644 index 00000000000..449b1e88e24 --- /dev/null +++ b/src/lib/pkcs5.c @@ -0,0 +1,95 @@ +#include "lib.h" +#include "buffer.h" +#include "hash-method.h" +#include "hmac.h" +#include "pkcs5.h" + +#include +#include + +static +int pkcs5_pbkdf1(const struct hash_method *hash, + const unsigned char *password, size_t password_len, + const unsigned char *salt, size_t salt_len, + unsigned int iter, uint32_t length, + buffer_t *result) +{ + if (length < 1 || + length > hash->digest_size) return -1; + if (iter < 1) return -1; + + unsigned char dk[hash->digest_size]; + unsigned char ctx[hash->context_size]; + + hash->init(ctx); + hash->loop(ctx, password, password_len); + hash->loop(ctx, salt, salt_len); + hash->result(ctx, dk); + length--; + + for(;length>0;length--) { + hash->init(ctx); + hash->loop(ctx, dk, hash->digest_size); + hash->result(ctx, dk); + } + + buffer_append(result, dk, hash->digest_size); + + return 0; +} + +static +int pkcs5_pbkdf2(const struct hash_method *hash, + const unsigned char *password, size_t password_len, + const unsigned char *salt, size_t salt_len, + unsigned int iter, uint32_t length, + buffer_t *result) +{ + if (length < 1 || iter < 1) return -1; + + size_t l = (length + hash->digest_size - 1)/hash->digest_size; /* same as ceil(length/hash->digest_size) */ + unsigned char dk[l * hash->digest_size]; + unsigned char *block; + struct hmac_context hctx; + unsigned int c,i,t; + unsigned char U_c[hash->digest_size]; + + for(t = 0; t < l; t++) { + block = &(dk[t*hash->digest_size]); + /* U_1 = PRF(Password, Salt|| INT_BE32(Block_Number)) */ + c = htonl(t+1); + hmac_init(&hctx, password, password_len, hash); + hmac_update(&hctx, salt, salt_len); + hmac_update(&hctx, &c, sizeof(c)); + hmac_final(&hctx, U_c); + /* block = U_1 ^ .. ^ U_iter */ + memcpy(block, U_c, hash->digest_size); + /* U_c = PRF(Password, U_c-1) */ + for(c = 1; c < iter; c++) { + hmac_init(&hctx, password, password_len, hash); + hmac_update(&hctx, U_c, hash->digest_size); + hmac_final(&hctx, U_c); + for(i = 0; i < hash->digest_size; i++) + block[i] ^= U_c[i]; + } + } + + buffer_append(result, dk, length); + + return 0; +} + +int pkcs5_pbkdf(enum pkcs5_pbkdf_mode mode, const struct hash_method *hash, + const unsigned char *password, size_t password_len, + const unsigned char *salt, size_t salt_len, + unsigned int iterations, uint32_t dk_len, + buffer_t *result) +{ + if (mode == PKCS5_PBKDF1) + return pkcs5_pbkdf1(hash,password,password_len, + salt,salt_len,iterations,dk_len,result); + else if (mode == PKCS5_PBKDF2) + return pkcs5_pbkdf2(hash,password,password_len, + salt,salt_len,iterations,dk_len,result); + i_unreached(); +} diff --git a/src/lib/pkcs5.h b/src/lib/pkcs5.h new file mode 100644 index 00000000000..4a249d48da4 --- /dev/null +++ b/src/lib/pkcs5.h @@ -0,0 +1,35 @@ +#ifndef PKCS5_H +#define PKCS5_H 1 + +enum pkcs5_pbkdf_mode { + PKCS5_PBKDF1, + PKCS5_PBKDF2 +}; + +/* + + mode - v1.0 or v2.0 + hash - hash_method_lookup return value + password - private password for generation + password_len - length of password in octets + salt - salt for generation + salt_len - length of salt in octets + iterations - number of iterations to hash (use at least 1000, a very large number => very very slow) + dk_len - number of bytes to return from derived key + result - buffer_t to hold the result, either use dynamic or make sure it fits dk_len + + non-zero return value indicates that either iterations was less than 1 or dk_len was too large + + Sample code: + + buffer_t *result = buffer_create_dynamic(pool_datastack_create(), 256); + if (pkcs5_pbkdf(PKCS5_PBKDF2, hash_method_lookup("sha256"), "password", 8, "salt", 4, 4096, 256, result) != 0) { // error } + +*/ + +int pkcs5_pbkdf(enum pkcs5_pbkdf_mode mode, const struct hash_method *hash, + const unsigned char *password, size_t password_len, + const unsigned char *salt, size_t salt_len, + unsigned int iterations, uint32_t dk_len, + buffer_t *result); +#endif diff --git a/src/lib/sha1.c b/src/lib/sha1.c index 947f3f7d8fb..edb1735e7fb 100644 --- a/src/lib/sha1.c +++ b/src/lib/sha1.c @@ -253,7 +253,7 @@ sha1_result(struct sha1_ctxt *ctxt, void *digest0) } void sha1_get_digest(const void *data, size_t size, - unsigned char result[SHA1_RESULTLEN]) + unsigned char result[STATIC_ARRAY SHA1_RESULTLEN]) { struct sha1_ctxt ctx; diff --git a/src/lib/sha1.h b/src/lib/sha1.h index 539f3c50c53..73b3110582c 100644 --- a/src/lib/sha1.h +++ b/src/lib/sha1.h @@ -77,7 +77,7 @@ typedef struct sha1_ctxt SHA1_CTX; #define SHA1_RESULTLEN (160/8) extern void sha1_get_digest(const void *, size_t, - unsigned char [SHA1_RESULTLEN]); + unsigned char [STATIC_ARRAY SHA1_RESULTLEN]); extern const struct hash_method hash_method_sha1; diff --git a/src/lib/sha2.c b/src/lib/sha2.c index 9ea500d6f09..c3aa91dee54 100644 --- a/src/lib/sha2.c +++ b/src/lib/sha2.c @@ -265,7 +265,7 @@ void sha256_loop(struct sha256_ctx *ctx, const void *data, } void sha256_result(struct sha256_ctx *ctx, - unsigned char digest[SHA256_RESULTLEN]) + unsigned char digest[STATIC_ARRAY SHA256_RESULTLEN]) { size_t block_nb; size_t pm_len; @@ -290,7 +290,7 @@ void sha256_result(struct sha256_ctx *ctx, } void sha256_get_digest(const void *data, size_t size, - unsigned char digest[SHA256_RESULTLEN]) + unsigned char digest[STATIC_ARRAY SHA256_RESULTLEN]) { struct sha256_ctx ctx; @@ -390,7 +390,7 @@ void sha512_loop(struct sha512_ctx *ctx, const void *data, } void sha512_result(struct sha512_ctx *ctx, - unsigned char digest[SHA512_RESULTLEN]) + unsigned char digest[STATIC_ARRAY SHA512_RESULTLEN]) { unsigned int block_nb; unsigned int pm_len; @@ -415,7 +415,7 @@ void sha512_result(struct sha512_ctx *ctx, } void sha512_get_digest(const void *data, size_t size, - unsigned char digest[SHA512_RESULTLEN]) + unsigned char digest[STATIC_ARRAY SHA512_RESULTLEN]) { struct sha512_ctx ctx; diff --git a/src/lib/sha2.h b/src/lib/sha2.h index 3ee937441d6..b04f93ae5a0 100644 --- a/src/lib/sha2.h +++ b/src/lib/sha2.h @@ -59,18 +59,18 @@ struct sha512_ctx { void sha256_init(struct sha256_ctx *ctx); void sha256_loop(struct sha256_ctx *ctx, const void *data, size_t len); void sha256_result(struct sha256_ctx *ctx, - unsigned char digest[SHA256_RESULTLEN]); + unsigned char digest[STATIC_ARRAY SHA256_RESULTLEN]); void sha256_get_digest(const void *data, size_t size, - unsigned char digest[SHA256_RESULTLEN]); + unsigned char digest[STATIC_ARRAY SHA256_RESULTLEN]); void sha512_init(struct sha512_ctx *ctx); void sha512_loop(struct sha512_ctx *ctx, const void *data, size_t len); void sha512_result(struct sha512_ctx *ctx, - unsigned char digest[SHA512_RESULTLEN]); + unsigned char digest[STATIC_ARRAY SHA512_RESULTLEN]); void sha512_get_digest(const void *data, size_t size, - unsigned char digest[SHA512_RESULTLEN]); + unsigned char digest[STATIC_ARRAY SHA512_RESULTLEN]); extern const struct hash_method hash_method_sha256; extern const struct hash_method hash_method_sha512; diff --git a/src/lib/str.c b/src/lib/str.c index 4203432aced..8cfd5d207d6 100644 --- a/src/lib/str.c +++ b/src/lib/str.c @@ -153,11 +153,11 @@ void str_vprintfa(string_t *str, const char *fmt, va_list args) init_size += SNPRINTF_INITIAL_EXTRA_SIZE; /* @UNSAFE */ - if (pos+init_size > buffer_get_size(str) && - pos < buffer_get_size(str)) { + if (pos+init_size > buffer_get_writable_size(str) && + pos < buffer_get_writable_size(str)) { /* avoid growing buffer larger if possible. this is also required if buffer isn't dynamically growing. */ - init_size = buffer_get_size(str)-pos; + init_size = buffer_get_writable_size(str)-pos; } tmp = buffer_get_space_unsafe(str, pos, init_size); ret = vsnprintf(tmp, init_size, fmt, args); diff --git a/src/lib/strescape.c b/src/lib/strescape.c index 065a50910b1..75641c21e36 100644 --- a/src/lib/strescape.c +++ b/src/lib/strescape.c @@ -226,6 +226,14 @@ char *str_tabunescape(char *str) return start; } +const char *t_str_tabunescape(const char *str) +{ + if (strchr(str, '\001') == NULL) + return str; + else + return str_tabunescape(t_strdup_noconst(str)); +} + char **p_strsplit_tabescaped(pool_t pool, const char *str) { char **args; diff --git a/src/lib/strescape.h b/src/lib/strescape.h index 9569b94fd45..550e3adfac3 100644 --- a/src/lib/strescape.h +++ b/src/lib/strescape.h @@ -23,6 +23,7 @@ const char *str_tabescape(const char *str); void str_append_tabescaped(string_t *dest, const char *src); void str_append_tabunescaped(string_t *dest, const void *src, size_t src_size); char *str_tabunescape(char *str); +const char *t_str_tabunescape(const char *str); char **p_strsplit_tabescaped(pool_t pool, const char *str); const char *const *t_strsplit_tabescaped(const char *str); diff --git a/src/lib/test-lib.c b/src/lib/test-lib.c index 325a37372c8..46f6f72fda9 100644 --- a/src/lib/test-lib.c +++ b/src/lib/test-lib.c @@ -39,6 +39,7 @@ int main(void) test_mempool_alloconly, test_net, test_numpack, + test_pkcs5_pbkdf2, test_ostream_escaped, test_ostream_failure_at, test_ostream_file, diff --git a/src/lib/test-lib.h b/src/lib/test-lib.h index e9cfe45ce08..23fc1eee6cf 100644 --- a/src/lib/test-lib.h +++ b/src/lib/test-lib.h @@ -39,6 +39,7 @@ void test_json_tree(void); void test_llist(void); void test_mempool_alloconly(void); enum fatal_test_state fatal_mempool(int); +void test_pkcs5_pbkdf2(void); void test_net(void); void test_numpack(void); void test_ostream_escaped(void); diff --git a/src/lib/test-ostream-escaped.c b/src/lib/test-ostream-escaped.c index f64813f4a0c..29d0235d9a8 100644 --- a/src/lib/test-ostream-escaped.c +++ b/src/lib/test-ostream-escaped.c @@ -24,6 +24,7 @@ static void test_ostream_escaped_json(void) iov[1].iov_len = 7; test_assert(o_stream_sendv(os_encode, iov, 2) == 12); test_assert(os_encode->offset == 12); + test_assert(os_sink->offset == 12); test_assert(strcmp(str_c(str), "hello, world") == 0); /* reset buffer */ @@ -33,7 +34,8 @@ static void test_ostream_escaped_json(void) o_stream_set_max_buffer_size(os_encode, 10); o_stream_set_max_buffer_size(os_sink, 100); test_assert(o_stream_send(os_encode, "\x15\x00!\x00\x15\x11" "123456", 12) == 12); - test_assert(os_encode->offset == 2*6 + 1 + 3*6 + 6); + test_assert(os_encode->offset == 12); + test_assert(os_sink->offset == 2*6 + 1 + 3*6 + 6); test_assert(strcmp(str_c(str), "\\u0015\\u0000!\\u0000\\u0015\\u0011123456") == 0); /* reset buffer */ @@ -49,7 +51,8 @@ static void test_ostream_escaped_json(void) o_stream_set_max_buffer_size(os_sink, 100); ret += o_stream_send_str(os_encode, partial_input + ret); test_assert(ret == (ssize_t)strlen(partial_input)); - test_assert(os_encode->offset == str_len(str)); + test_assert((ssize_t)os_encode->offset == ret); + test_assert(os_sink->offset == str_len(str)); test_assert(strcmp(str_c(str), "\\u0015!\\u0001?#&") == 0); o_stream_unref(&os_encode); diff --git a/src/lib/test-pkcs5.c b/src/lib/test-pkcs5.c new file mode 100644 index 00000000000..97b23b82501 --- /dev/null +++ b/src/lib/test-pkcs5.c @@ -0,0 +1,49 @@ +/* Copyright (c) 2007-2016 Dovecot authors, see the included COPYING file */ + +#include "test-lib.h" +#include "str.h" +#include "buffer.h" +#include "hash-method.h" +#include "pkcs5.h" + +struct test_vector { + const char *prf; + unsigned char *p; + size_t pLen; + unsigned char *s; + size_t sLen; + unsigned int i; + unsigned char *dk; + size_t dkLen; +}; + +#define TEST_BUF(x) (unsigned char*)x, sizeof(x)-1 + +/* RFC 6070 test vectors */ +static const struct test_vector test_vectors_v2[] = { + { "sha1", TEST_BUF("password"), TEST_BUF("salt"), 1, TEST_BUF("\x0c\x60\xc8\x0f\x96\x1f\x0e\x71\xf3\xa9\xb5\x24\xaf\x60\x12\x06\x2f\xe0\x37\xa6") }, + { "sha1", TEST_BUF("password"), TEST_BUF("salt"), 2, TEST_BUF("\xea\x6c\x01\x4d\xc7\x2d\x6f\x8c\xcd\x1e\xd9\x2a\xce\x1d\x41\xf0\xd8\xde\x89\x57") }, + { "sha1", TEST_BUF("password"), TEST_BUF("salt"), 4096, TEST_BUF("\x4b\x00\x79\x01\xb7\x65\x48\x9a\xbe\xad\x49\xd9\x26\xf7\x21\xd0\x65\xa4\x29\xc1") }, +/* enable the next test only when you need it, it takes quite long time */ +/* { "sha1", TEST_BUF("password"), TEST_BUF("salt"), 16777216, TEST_BUF("\xee\xfe\x3d\x61\xcd\x4d\xa4\xe4\xe9\x94\x5b\x3d\x6b\xa2\x15\x8c\x26\x34\xe9\x84") }, */ + { "sha1", TEST_BUF("passwordPASSWORDpassword"), TEST_BUF("saltSALTsaltSALTsaltSALTsaltSALTsalt"), 4096, TEST_BUF("\x3d\x2e\xec\x4f\xe4\x1c\x84\x9b\x80\xc8\xd8\x36\x62\xc0\xe4\x4a\x8b\x29\x1a\x96\x4c\xf2\xf0\x70\x38") }, + { "sha1", TEST_BUF("pass\0word"), TEST_BUF("sa\0lt"), 4096, TEST_BUF("\x56\xfa\x6a\xa7\x55\x48\x09\x9d\xcc\x37\xd7\xf0\x34\x25\xe0\xc3") } +}; + +void test_pkcs5_pbkdf2(void) +{ + buffer_t *res = buffer_create_dynamic(default_pool, 25); + + test_begin("pkcs5_pbkdf2"); + + for(size_t i = 0; i < N_ELEMENTS(test_vectors_v2); i++) { + buffer_reset(res); + const struct test_vector *vec = &(test_vectors_v2[i]); + pkcs5_pbkdf(PKCS5_PBKDF2, hash_method_lookup(vec->prf), vec->p, vec->pLen, vec->s, vec->sLen, vec->i, vec->dkLen, res); + test_assert_idx(memcmp(res->data, vec->dk, vec->dkLen) == 0, i); + } + + buffer_free(&res); + + test_end(); +} diff --git a/src/lib/test-strescape.c b/src/lib/test-strescape.c index c4c5c3905af..4272cae3b26 100644 --- a/src/lib/test-strescape.c +++ b/src/lib/test-strescape.c @@ -70,6 +70,8 @@ void test_strescape(void) test_begin("str_tabescape"); for (i = 0; i < N_ELEMENTS(tabesc); i++) { + test_assert(strcmp(t_str_tabunescape(tabesc[i].output), + tabesc[i].input) == 0); test_assert(strcmp(str_tabunescape(t_strdup_noconst(tabesc[i].output)), tabesc[i].input) == 0); test_assert(strcmp(str_tabescape(tabesc[i].input), diff --git a/src/lib/test-timing.c b/src/lib/test-timing.c index e83df9f9bcb..c79d144dc39 100644 --- a/src/lib/test-timing.c +++ b/src/lib/test-timing.c @@ -71,7 +71,25 @@ void test_timing(void) timing_add_usecs(t, test_inputs[i][j]); test_timing_verify(t, test_inputs[i], j+1); } + timing_reset(t); + test_assert(timing_get_count(t) == 0); + test_assert(timing_get_max(t) == 0); timing_deinit(&t); test_end(); } + + test_begin("timings large"); + t = timing_init(); + for (i = 0; i < 10000; i++) + timing_add_usecs(t, i); + test_assert(timing_get_count(t) == i); + test_assert(timing_get_sum(t) == (i-1)*i/2); + test_assert(timing_get_min(t) == 0); + test_assert(timing_get_max(t) == i-1); + test_assert(timing_get_avg(t) == i/2); + /* just test that these work: */ + test_assert(timing_get_median(t) > 0 && timing_get_median(t) < i-1); + test_assert(timing_get_95th(t) > 0 && timing_get_95th(t) < i-1); + timing_deinit(&t); + test_end(); } diff --git a/src/lib/test-var-expand.c b/src/lib/test-var-expand.c index 370199cb26c..d3f248a9ed4 100644 --- a/src/lib/test-var-expand.c +++ b/src/lib/test-var-expand.c @@ -52,7 +52,10 @@ static void test_var_expand_builtin(void) { "%50Hv", "1f" }, { "%50Hw", "2e" }, { "%50Nv", "25" }, - { "%50Nw", "e" } + { "%50Nw", "e" }, + + { "%{nonexistent}", "UNSUPPORTED_VARIABLE_nonexistent" }, + { "%{nonexistent:default}", "UNSUPPORTED_VARIABLE_nonexistent" }, }; static struct var_expand_table table[] = { { 'v', "value", NULL }, diff --git a/src/lib/time-util.c b/src/lib/time-util.c index 9bfad68558c..3ab1a86821a 100644 --- a/src/lib/time-util.c +++ b/src/lib/time-util.c @@ -28,10 +28,10 @@ int timeval_cmp_margin(const struct timeval *tv1, const struct timeval *tv2, if (tv1->tv_sec > tv2->tv_sec) return 1; - if (tv1->tv_usec - tv2->tv_usec < (int)usec_margin) - return -1; - if (tv1->tv_usec - tv2->tv_usec > (int)usec_margin) + if ((tv2->tv_usec - tv1->tv_usec) > (int)usec_margin) return -1; + if ((tv1->tv_usec - tv2->tv_usec) > (int)usec_margin) + return 1; return 0; } diff --git a/src/lib/timing.c b/src/lib/timing.c index 18838c474e2..604f102a3f4 100644 --- a/src/lib/timing.c +++ b/src/lib/timing.c @@ -27,6 +27,11 @@ void timing_deinit(struct timing **_timing) i_free_and_null(*_timing); } +void timing_reset(struct timing *timing) +{ + memset(timing, 0, sizeof(*timing)); +} + void timing_add_usecs(struct timing *timing, uint64_t usecs) { if (timing->count < TIMING_SUBSAMPLING_BUFFER) { @@ -85,7 +90,11 @@ static void timing_ensure_sorted(struct timing *timing) { if (timing->sorted) return; - i_qsort(timing->samples, timing->count, sizeof(*timing->samples), + + unsigned int count = (timing->count < TIMING_SUBSAMPLING_BUFFER) + ? timing->count + : TIMING_SUBSAMPLING_BUFFER; + i_qsort(timing->samples, count, sizeof(*timing->samples), uint64_cmp); timing->sorted = TRUE; } diff --git a/src/lib/timing.h b/src/lib/timing.h index 413dc2ed001..79536f9ed19 100644 --- a/src/lib/timing.h +++ b/src/lib/timing.h @@ -4,6 +4,9 @@ struct timing *timing_init(void); void timing_deinit(struct timing **timing); +/* Reset all events. */ +void timing_reset(struct timing *timing); + /* Add a new event that took the specified number of usecs. */ void timing_add_usecs(struct timing *timing, uint64_t usecs); diff --git a/src/lib/utc-mktime.c b/src/lib/utc-mktime.c index 89522579bc3..566cb44dc26 100644 --- a/src/lib/utc-mktime.c +++ b/src/lib/utc-mktime.c @@ -20,6 +20,18 @@ static int tm_cmp(const struct tm *tm1, const struct tm *tm2) return tm1->tm_sec - tm2->tm_sec; } +#ifdef HAVE_TIMEGM +time_t utc_mktime(const struct tm *tm) +{ + struct tm mod_tm = *tm; + time_t t; + + t = timegm(&mod_tm); + if (tm_cmp(tm, &mod_tm) != 0) + return (time_t)-1; + return t; +} +#else time_t utc_mktime(const struct tm *tm) { const struct tm *try_tm; @@ -51,3 +63,4 @@ time_t utc_mktime(const struct tm *tm) return (time_t)-1; } +#endif diff --git a/src/lib/var-expand.c b/src/lib/var-expand.c index fee44c9c4bc..5d06cebee05 100644 --- a/src/lib/var-expand.c +++ b/src/lib/var-expand.c @@ -232,6 +232,8 @@ var_expand_long(const struct var_expand_table *table, data = ""; value = var_expand_func(func_table, key, data, context); } + if (value == NULL) + return t_strdup_printf("UNSUPPORTED_VARIABLE_%s", key); return value; } @@ -325,8 +327,8 @@ void var_expand_with_funcs(string_t *dest, const char *str, len = end - (str + 1); var = var_expand_long(table, func_table, str+1, len, context); - if (var != NULL) - str = end; + i_assert(var != NULL); + str = end; } else if (table != NULL) { for (t = table; !TABLE_LAST(t); t++) { if (t->key == *str) { diff --git a/src/lmtp/client.c b/src/lmtp/client.c index 8f9d704c998..5650b412123 100644 --- a/src/lmtp/client.c +++ b/src/lmtp/client.c @@ -343,6 +343,19 @@ void client_disconnect(struct client *client, const char *prefix, client->disconnected = TRUE; } +void client_rcpt_anvil_disconnect(const struct mail_recipient *rcpt) +{ + const struct mail_storage_service_input *input; + + if (!rcpt->anvil_connect_sent) + return; + + input = mail_storage_service_user_get_input(rcpt->service_user); + master_service_anvil_send(master_service, t_strconcat( + "DISCONNECT\t", my_pid, "\t", master_service_get_name(master_service), + "/", input->username, "\n", NULL)); +} + void client_state_reset(struct client *client, const char *state_name) { struct mail_recipient *const *rcptp; @@ -354,6 +367,7 @@ void client_state_reset(struct client *client, const char *state_name) array_foreach_modifiable(&client->state.rcpt_to, rcptp) { if ((*rcptp)->anvil_query != NULL) anvil_client_query_abort(anvil, &(*rcptp)->anvil_query); + client_rcpt_anvil_disconnect(*rcptp); mail_storage_service_user_free(&(*rcptp)->service_user); } } diff --git a/src/lmtp/client.h b/src/lmtp/client.h index c5b8b441353..fe91244a6e0 100644 --- a/src/lmtp/client.h +++ b/src/lmtp/client.h @@ -15,8 +15,8 @@ struct mail_recipient { struct lmtp_recipient_params params; struct anvil_query *anvil_query; + bool anvil_connect_sent; struct mail_storage_service_user *service_user; - unsigned int parallel_count; }; struct client_state { @@ -26,9 +26,6 @@ struct client_state { ARRAY(struct mail_recipient *) rcpt_to; unsigned int rcpt_idx; - unsigned int anvil_queries; - bool anvil_pending_data_write; - unsigned int data_end_idx; /* Initially we start writing to mail_data. If it grows too large, @@ -92,6 +89,7 @@ void client_destroy(struct client *client, const char *prefix, void client_disconnect(struct client *client, const char *prefix, const char *reason); void client_io_reset(struct client *client); +void client_rcpt_anvil_disconnect(const struct mail_recipient *rcpt); void client_state_reset(struct client *client, const char *state_name); void client_state_set(struct client *client, const char *name, const char *args); const char *client_remote_id(struct client *client); diff --git a/src/lmtp/commands.c b/src/lmtp/commands.c index 480e9f2d53b..480c28dc43d 100644 --- a/src/lmtp/commands.c +++ b/src/lmtp/commands.c @@ -41,8 +41,6 @@ #define LMTP_PROXY_DEFAULT_TIMEOUT_MSECS (1000*125) -static void client_input_data_write(struct client *client); - int cmd_lhlo(struct client *client, const char *args) { struct rfc822_parser_context parser; @@ -179,6 +177,14 @@ parse_xtext(struct client *client, const char *value) return p_strdup(client->state_pool, str_c(str)); } +static void lmtp_anvil_init(void) +{ + if (anvil == NULL) { + const char *path = t_strdup_printf("%s/anvil", base_dir); + anvil = anvil_client_init(path, NULL, 0); + } +} + int cmd_mail(struct client *client, const char *args) { const char *addr, *const *argv; @@ -212,6 +218,11 @@ int cmd_mail(struct client *client, const char *args) client_send_line(client, "250 2.1.0 OK"); client_state_set(client, "MAIL FROM", client->state.mail_from); + if (client->lmtp_set->lmtp_user_concurrency_limit > 0) { + /* connect to anvil before dropping privileges */ + lmtp_anvil_init(); + } + client->state.mail_from_timeval = ioloop_timeval; return 0; } @@ -587,29 +598,35 @@ lmtp_rcpt_to_is_over_quota(struct client *client, static void rcpt_anvil_lookup_callback(const char *reply, void *context) { struct mail_recipient *rcpt = context; - - i_assert(rcpt->client->state.anvil_queries > 0); + struct client *client = rcpt->client; + const struct mail_storage_service_input *input; + unsigned int parallel_count = 0; rcpt->anvil_query = NULL; if (reply == NULL) { /* lookup failed */ - } else if (str_to_uint(reply, &rcpt->parallel_count) < 0) { + } else if (str_to_uint(reply, ¶llel_count) < 0) { i_error("Invalid reply from anvil: %s", reply); } - if (--rcpt->client->state.anvil_queries == 0 && - rcpt->client->state.anvil_pending_data_write) { - /* DATA command was finished, but we were still waiting on - anvil before handling any users */ - client_input_data_write(rcpt->client); - } -} -static void lmtp_anvil_init(void) -{ - if (anvil == NULL) { - const char *path = t_strdup_printf("%s/anvil", base_dir); - anvil = anvil_client_init(path, NULL, 0); + if (parallel_count < client->lmtp_set->lmtp_user_concurrency_limit) { + client_send_line(client, "250 2.1.5 OK"); + + rcpt->anvil_connect_sent = TRUE; + input = mail_storage_service_user_get_input(rcpt->service_user); + master_service_anvil_send(master_service, t_strconcat( + "CONNECT\t", my_pid, "\t", master_service_get_name(master_service), + "/", input->username, "\n", NULL)); + array_append(&client->state.rcpt_to, &rcpt, 1); + } else { + client_send_line(client, ERRSTR_TEMP_USERDB_FAIL_PREFIX + "Too many concurrent deliveries for user", + rcpt->address); + mail_storage_service_user_free(&rcpt->service_user); } + + client_io_reset(client); + client_input_handle(client); } int cmd_rcpt(struct client *client, const char *args) @@ -713,19 +730,22 @@ int cmd_rcpt(struct client *client, const char *args) mail_storage_service_user_free(&rcpt->service_user); return 0; } - array_append(&client->state.rcpt_to, &rcpt, 1); - client_send_line(client, "250 2.1.5 OK"); - if (client->lmtp_set->lmtp_user_concurrency_limit > 0) { + if (client->lmtp_set->lmtp_user_concurrency_limit == 0) { + array_append(&client->state.rcpt_to, &rcpt, 1); + client_send_line(client, "250 2.1.5 OK"); + return 0; + } else { const char *query = t_strconcat("LOOKUP\t", master_service_get_name(master_service), "/", str_tabescape(username), NULL); - lmtp_anvil_init(); - client->state.anvil_queries++; + io_remove(&client->io); rcpt->anvil_query = anvil_client_query(anvil, query, rcpt_anvil_lookup_callback, rcpt); + /* stop processing further commands while anvil query is + pending */ + return rcpt->anvil_query == NULL ? 0 : -1; } - return 0; } int cmd_quit(struct client *client, const char *args ATTR_UNUSED) @@ -784,19 +804,9 @@ client_deliver(struct client *client, const struct mail_recipient *rcpt, enum mail_error mail_error; int ret; - i_assert(client->state.anvil_queries == 0); - input = mail_storage_service_user_get_input(rcpt->service_user); username = t_strdup(input->username); - if (client->lmtp_set->lmtp_user_concurrency_limit > 0 && - rcpt->parallel_count >= client->lmtp_set->lmtp_user_concurrency_limit) { - client_send_line(client, ERRSTR_TEMP_USERDB_FAIL_PREFIX - "Too many concurrent deliveries for user", - rcpt->address); - return -1; - } - mail_set = mail_storage_service_user_get_mail_set(rcpt->service_user); set_parser = mail_storage_service_user_get_settings_parser(rcpt->service_user); if (client->proxy_timeout_secs > 0 && @@ -870,11 +880,6 @@ client_deliver(struct client *client, const struct mail_recipient *rcpt, dctx.save_dest_mail = array_count(&client->state.rcpt_to) > 1 && client->state.first_saved_mail == NULL; - if (client->lmtp_set->lmtp_user_concurrency_limit > 0) { - master_service_anvil_send(master_service, t_strconcat( - "CONNECT\t", my_pid, "\t", master_service_get_name(master_service), - "/", username, "\n", NULL)); - } if (mail_deliver(&dctx, &storage) == 0) { if (dctx.dest_mail != NULL) { i_assert(client->state.first_saved_mail == NULL); @@ -903,11 +908,7 @@ client_deliver(struct client *client, const struct mail_recipient *rcpt, rcpt->address); ret = -1; } - if (client->lmtp_set->lmtp_user_concurrency_limit > 0) { - master_service_anvil_send(master_service, t_strconcat( - "DISCONNECT\t", my_pid, "\t", master_service_get_name(master_service), - "/", username, "\n", NULL)); - } + client_rcpt_anvil_disconnect(rcpt); return ret; } @@ -1167,7 +1168,6 @@ static int client_input_add_file(struct client *client, { struct client_state *state = &client->state; string_t *path; - ssize_t ret; int fd; if (state->mail_data_output != NULL) { @@ -1196,15 +1196,17 @@ static int client_input_add_file(struct client *client, state->mail_data_fd = fd; state->mail_data_output = o_stream_create_fd_file(fd, 0, FALSE); + o_stream_set_name(state->mail_data_output, str_c(path)); o_stream_cork(state->mail_data_output); - ret = o_stream_send(state->mail_data_output, - state->mail_data->data, state->mail_data->used); - if (ret != (ssize_t)state->mail_data->used) - return -1; - if (o_stream_send(client->state.mail_data_output, - data, size) != (ssize_t)size) + o_stream_nsend(state->mail_data_output, + state->mail_data->data, state->mail_data->used); + o_stream_nsend(client->state.mail_data_output, data, size); + if (o_stream_nfinish(client->state.mail_data_output) < 0) { + i_error("write(%s) failed: %s", str_c(path), + o_stream_get_error(client->state.mail_data_output)); return -1; + } return 0; } @@ -1245,10 +1247,7 @@ static void client_input_data_handle(struct client *client) return; } - if (client->state.anvil_queries == 0) - client_input_data_write(client); - else - client->state.anvil_pending_data_write = TRUE; + client_input_data_write(client); } static void client_input_data(struct client *client) diff --git a/src/login-common/login-proxy.c b/src/login-common/login-proxy.c index 85e40df03e8..e24d8eb032b 100644 --- a/src/login-common/login-proxy.c +++ b/src/login-common/login-proxy.c @@ -197,10 +197,12 @@ static void proxy_client_disconnected_input(struct login_proxy *proxy) static int server_output(struct login_proxy *proxy) { proxy->last_io = ioloop_time; + o_stream_cork(proxy->server_output); if (o_stream_flush(proxy->server_output) < 0) { login_proxy_free_ostream(&proxy, proxy->server_output, TRUE); return 1; } + o_stream_uncork(proxy->server_output); if (proxy->client_io == NULL && o_stream_get_buffer_used_size(proxy->server_output) < @@ -216,10 +218,12 @@ static int server_output(struct login_proxy *proxy) static int proxy_client_output(struct login_proxy *proxy) { proxy->last_io = ioloop_time; + o_stream_cork(proxy->client_output); if (o_stream_flush(proxy->client_output) < 0) { login_proxy_free_ostream(&proxy, proxy->client_output, FALSE); return 1; } + o_stream_uncork(proxy->client_output); if (proxy->server_io == NULL && o_stream_get_buffer_used_size(proxy->client_output) < diff --git a/src/master/Makefile.am b/src/master/Makefile.am index 9d3ad7844f0..ef7955ff315 100644 --- a/src/master/Makefile.am +++ b/src/master/Makefile.am @@ -25,6 +25,7 @@ dovecot_SOURCES = \ capabilities-posix.c \ dup2-array.c \ main.c \ + master-client.c \ master-settings.c \ service-anvil.c \ service-listen.c \ @@ -39,6 +40,7 @@ noinst_HEADERS = \ capabilities.h \ common.h \ dup2-array.h \ + master-client.h \ master-settings.h \ sd-daemon.h \ service-anvil.h \ diff --git a/src/master/main.c b/src/master/main.c index a98a0b4c56f..8fd4cb77d27 100644 --- a/src/master/main.c +++ b/src/master/main.c @@ -18,6 +18,7 @@ #include "master-service-settings.h" #include "askpass.h" #include "capabilities.h" +#include "master-client.h" #include "service.h" #include "service-anvil.h" #include "service-listen.h" @@ -526,6 +527,7 @@ static void main_init(const struct master_settings *set) create_pid_file(pidfile_path); create_config_symlink(set); instance_update(set); + master_clients_init(); services_monitor_start(services); } @@ -542,6 +544,7 @@ static void global_dead_pipe_close(void) static void main_deinit(void) { + master_clients_deinit(); instance_update_now(instances); timeout_remove(&to_instance); master_instance_list_deinit(&instances); diff --git a/src/master/master-client.c b/src/master/master-client.c new file mode 100644 index 00000000000..32cfbc3eb4f --- /dev/null +++ b/src/master/master-client.c @@ -0,0 +1,100 @@ +/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */ + +#include "common.h" +#include "ostream.h" +#include "connection.h" +#include "service.h" +#include "service-monitor.h" +#include "master-client.h" + +struct master_client { + struct connection conn; +}; + +static int +master_client_stop(struct master_client *client, const char *const *args) +{ + struct service *service; + const char *reply = "+\n"; + + for (unsigned int i = 0; args[i] != NULL; i++) { + service = service_lookup(services, args[i]); + if (service == NULL) + reply = t_strdup_printf("-Unknown service: %s\n", args[i]); + else + service_monitor_stop_close(service); + } + o_stream_send_str(client->conn.output, reply); + return 1; +} + +static int +master_client_input_args(struct connection *conn, const char *const *args) +{ + struct master_client *client = (struct master_client *)conn; + const char *cmd = args[0]; + + if (cmd == NULL) { + i_error("%s: Empty command", conn->name); + return 0; + } + args++; + + if (strcmp(cmd, "STOP") == 0) + return master_client_stop(client, args); + i_error("%s: Unknown command: %s", conn->name, cmd); + return -1; +} + +static void master_client_destroy(struct connection *conn) +{ + struct master_client *client = (struct master_client *)conn; + + connection_deinit(conn); + i_free(client); +} + +static const struct connection_settings master_conn_set = { + .service_name_in = "master-client", + .service_name_out = "master-server", + .major_version = 1, + .minor_version = 0, + + .input_max_size = 1024, + .output_max_size = 1024, + .client = FALSE +}; + +static const struct connection_vfuncs master_conn_vfuncs = { + .destroy = master_client_destroy, + .input_args = master_client_input_args +}; + +static struct connection_list *master_connections; + +void master_client_connected(struct service_list *service_list) +{ + struct master_client *client; + int fd; + + fd = net_accept(service_list->master_fd, NULL, NULL); + if (fd < 0) { + if (fd == -2) + i_error("net_accept() failed: %m"); + return; + } + client = i_new(struct master_client, 1); + connection_init_server(master_connections, &client->conn, + "master-client", fd, fd); +} + +void master_clients_init(void) +{ + master_connections = connection_list_init(&master_conn_set, + &master_conn_vfuncs); +} + +void master_clients_deinit(void) +{ + connection_list_deinit(&master_connections); +} diff --git a/src/master/master-client.h b/src/master/master-client.h new file mode 100644 index 00000000000..7e62edee8a6 --- /dev/null +++ b/src/master/master-client.h @@ -0,0 +1,9 @@ +#ifndef MASTER_CLIENT_H +#define MASTER_CLIENT_H + +void master_client_connected(struct service_list *service_list); + +void master_clients_init(void); +void master_clients_deinit(void); + +#endif diff --git a/src/master/master-settings.c b/src/master/master-settings.c index 101222d9dde..216eb07be5e 100644 --- a/src/master/master-settings.c +++ b/src/master/master-settings.c @@ -277,21 +277,27 @@ expand_user(const char **user, enum service_user_default *default_r, } } -static void +static bool fix_file_listener_paths(ARRAY_TYPE(file_listener_settings) *l, pool_t pool, const struct master_settings *master_set, - ARRAY_TYPE(const_string) *all_listeners) + ARRAY_TYPE(const_string) *all_listeners, + const char **error_r) { struct file_listener_settings *const *sets; unsigned int base_dir_len = strlen(master_set->base_dir); enum service_user_default user_default; if (!array_is_created(l)) - return; + return TRUE; array_foreach(l, sets) { struct file_listener_settings *set = *sets; + if (set->path[0] == '\0') { + *error_r = "path must not be empty"; + return FALSE; + } + expand_user(&set->user, &user_default, master_set); if (*set->path != '/') { set->path = p_strconcat(pool, master_set->base_dir, "/", @@ -305,6 +311,7 @@ fix_file_listener_paths(ARRAY_TYPE(file_listener_settings) *l, if (set->mode != 0) array_append(all_listeners, &set->path, 1); } + return TRUE; } static void add_inet_listeners(ARRAY_TYPE(inet_listener_settings) *l, @@ -425,6 +432,11 @@ master_settings_verify(void *_set, pool_t pool, const char **error_r) unsigned int max_client_limit = set->default_client_limit; #endif + if (*set->listen == '\0') { + *error_r = "listen can't be set empty"; + return FALSE; + } + len = strlen(set->base_dir); if (len > 0 && set->base_dir[len-1] == '/') { /* drop trailing '/' */ @@ -570,10 +582,18 @@ master_settings_verify(void *_set, pool_t pool, const char **error_r) strcmp(service->name, "auth") == 0) max_anvil_client_processes += process_limit; - fix_file_listener_paths(&service->unix_listeners, - pool, set, &all_listeners); - fix_file_listener_paths(&service->fifo_listeners, - pool, set, &all_listeners); + if (!fix_file_listener_paths(&service->unix_listeners, pool, + set, &all_listeners, error_r)) { + *error_r = t_strdup_printf("service(%s): unix_listener: %s", + service->name, *error_r); + return FALSE; + } + if (!fix_file_listener_paths(&service->fifo_listeners, pool, + set, &all_listeners, error_r)) { + *error_r = t_strdup_printf("service(%s): fifo_listener: %s", + service->name, *error_r); + return FALSE; + } add_inet_listeners(&service->inet_listeners, &all_listeners); } diff --git a/src/master/service-listen.c b/src/master/service-listen.c index 06da1f184ba..677b2204613 100644 --- a/src/master/service-listen.c +++ b/src/master/service-listen.c @@ -4,7 +4,9 @@ #include "array.h" #include "fd-set-nonblock.h" #include "fd-close-on-exec.h" +#include "ioloop.h" #include "net.h" +#include "master-client.h" #ifdef HAVE_SYSTEMD #include "sd-daemon.h" #endif @@ -337,6 +339,31 @@ static int services_verify_systemd(struct service_list *service_list) } #endif +static int services_listen_master(struct service_list *service_list) +{ + const char *path; + mode_t old_umask; + + path = t_strdup_printf("%s/master", service_list->set->base_dir); + old_umask = umask(0600 ^ 0777); + service_list->master_fd = net_listen_unix(path, 16); + if (service_list->master_fd == -1 && errno == EADDRINUSE) { + /* already in use. all the other sockets were fine, so just + delete this and retry. */ + i_unlink_if_exists(path); + service_list->master_fd = net_listen_unix(path, 16); + } + umask(old_umask); + + if (service_list->master_fd == -1) + return 0; + + service_list->io_master = + io_add(service_list->master_fd, IO_READ, + master_client_connected, service_list); + return 1; +} + int services_listen(struct service_list *service_list) { struct service *const *services; @@ -347,6 +374,8 @@ int services_listen(struct service_list *service_list) if (ret2 < ret) ret = ret2; } + if (ret > 0) + ret = services_listen_master(service_list); #ifdef HAVE_SYSTEMD if (ret > 0) diff --git a/src/master/service-monitor.c b/src/master/service-monitor.c index bf4e9312410..44cdb2696b2 100644 --- a/src/master/service-monitor.c +++ b/src/master/service-monitor.c @@ -452,11 +452,6 @@ void services_monitor_start(struct service_list *service_list) return; service_anvil_monitor_start(service_list); - if (pipe(service_list->master_dead_pipe_fd) < 0) - i_error("pipe() failed: %m"); - fd_close_on_exec(service_list->master_dead_pipe_fd[0], TRUE); - fd_close_on_exec(service_list->master_dead_pipe_fd[1], TRUE); - array_foreach(&service_list->services, services) { struct service *service = *services; @@ -464,6 +459,14 @@ void services_monitor_start(struct service_list *service_list) if (service_login_create_notify_fd(service) < 0) continue; } + if (service->master_dead_pipe_fd[0] == -1) { + if (pipe(service->master_dead_pipe_fd) < 0) { + service_error(service, "pipe() failed: %m"); + continue; + } + fd_close_on_exec(service->master_dead_pipe_fd[0], TRUE); + fd_close_on_exec(service->master_dead_pipe_fd[1], TRUE); + } if (service->status_fd[0] == -1) { /* we haven't yet created status pipe */ if (pipe(service->status_fd) < 0) { @@ -502,6 +505,14 @@ void services_monitor_start(struct service_list *service_list) } } +static void service_monitor_close_dead_pipe(struct service *service) +{ + if (service->master_dead_pipe_fd[0] != -1) { + i_close_fd(&service->master_dead_pipe_fd[0]); + i_close_fd(&service->master_dead_pipe_fd[1]); + } +} + void service_monitor_stop(struct service *service) { int i; @@ -519,6 +530,7 @@ void service_monitor_stop(struct service *service) service->status_fd[i] = -1; } } + service_monitor_close_dead_pipe(service); if (service->login_notify_fd != -1) { if (close(service->login_notify_fd) < 0) { service_error(service, @@ -536,6 +548,20 @@ void service_monitor_stop(struct service *service) timeout_remove(&service->to_prefork); } +void service_monitor_stop_close(struct service *service) +{ + struct service_listener *const *listeners; + + service_monitor_stop(service); + + array_foreach(&service->listeners, listeners) { + struct service_listener *l = *listeners; + + if (l->fd != -1) + i_close_fd(&l->fd); + } +} + static void services_monitor_wait(struct service_list *service_list) { struct service *const *servicep; @@ -561,14 +587,8 @@ void services_monitor_stop(struct service_list *service_list, bool wait) { struct service *const *services; - if (service_list->master_dead_pipe_fd[0] != -1) { - if (close(service_list->master_dead_pipe_fd[0]) < 0) - i_error("close(master dead pipe) failed: %m"); - if (close(service_list->master_dead_pipe_fd[1]) < 0) - i_error("close(master dead pipe) failed: %m"); - service_list->master_dead_pipe_fd[0] = -1; - service_list->master_dead_pipe_fd[1] = -1; - } + array_foreach(&service_list->services, services) + service_monitor_close_dead_pipe(*services); if (wait) { /* we've notified all children that the master is dead. @@ -577,6 +597,11 @@ void services_monitor_stop(struct service_list *service_list, bool wait) services_monitor_wait(service_list); } + if (service_list->io_master != NULL) + io_remove(&service_list->io_master); + if (service_list->master_fd != -1) + i_close_fd(&service_list->master_fd); + array_foreach(&service_list->services, services) service_monitor_stop(*services); diff --git a/src/master/service-monitor.h b/src/master/service-monitor.h index 848cb3c1e0e..7dd1c4db91f 100644 --- a/src/master/service-monitor.h +++ b/src/master/service-monitor.h @@ -11,6 +11,7 @@ void services_monitor_stop(struct service_list *service_list, bool wait); void services_monitor_reap_children(void); void service_monitor_stop(struct service *service); +void service_monitor_stop_close(struct service *service); void service_monitor_listen_start(struct service *service); void service_monitor_listen_stop(struct service *service); diff --git a/src/master/service-process.c b/src/master/service-process.c index d0f1df69a12..7a24c58394c 100644 --- a/src/master/service-process.c +++ b/src/master/service-process.c @@ -133,7 +133,7 @@ service_dup_fds(struct service *service) } dup2_append(&dups, service->status_fd[1], MASTER_STATUS_FD); if (service->type != SERVICE_TYPE_ANVIL) { - dup2_append(&dups, service->list->master_dead_pipe_fd[1], + dup2_append(&dups, service->master_dead_pipe_fd[1], MASTER_DEAD_FD); } else { dup2_append(&dups, global_master_dead_pipe_fd[1], diff --git a/src/master/service.c b/src/master/service.c index d9f49d6bb72..d175b0c4b95 100644 --- a/src/master/service.c +++ b/src/master/service.c @@ -283,6 +283,8 @@ service_create(pool_t pool, const struct service_settings *set, service->log_fd[1] = -1; service->status_fd[0] = -1; service->status_fd[1] = -1; + service->master_dead_pipe_fd[0] = -1; + service->master_dead_pipe_fd[1] = -1; service->log_process_internal_fd = -1; service->login_notify_fd = -1; @@ -424,8 +426,6 @@ services_create_real(const struct master_settings *set, pool_t pool, service_list->set = set; service_list->master_log_fd[0] = -1; service_list->master_log_fd[1] = -1; - service_list->master_dead_pipe_fd[0] = -1; - service_list->master_dead_pipe_fd[1] = -1; service_settings = array_get(&set->services, &count); p_array_init(&service_list->services, pool, count); diff --git a/src/master/service.h b/src/master/service.h index 79aa65f9ef0..6b52a228e6f 100644 --- a/src/master/service.h +++ b/src/master/service.h @@ -85,6 +85,8 @@ struct service { int status_fd[2]; struct io *io_status; + int master_dead_pipe_fd[2]; + unsigned int throttle_secs; time_t exit_failure_last; unsigned int exit_failures_in_sec; @@ -135,12 +137,14 @@ struct service_list { struct service *log; struct service *anvil; + struct file_listener_settings master_listener_set; + struct io *io_master; + int master_fd; + /* nonblocking log fds usd by master */ int master_log_fd[2]; struct service_process_notify *log_byes; - int master_dead_pipe_fd[2]; - ARRAY(struct service *) services; unsigned int destroying:1; diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am index b9196b38f73..7dc31d954c6 100644 --- a/src/plugins/Makefile.am +++ b/src/plugins/Makefile.am @@ -10,6 +10,10 @@ if BUILD_SOLR FTS_SOLR = fts-solr endif +if HAVE_LDAP +DICT_LDAP = dict-ldap +endif + SUBDIRS = \ acl \ imap-acl \ @@ -35,7 +39,9 @@ SUBDIRS = \ imap-stats \ trash \ virtual \ + welcome \ $(ZLIB) \ $(FTS_LUCENE) \ $(FTS_SOLR) \ + $(DICT_LDAP) \ fs-compress diff --git a/src/plugins/acl/acl-backend-vfile-acllist.c b/src/plugins/acl/acl-backend-vfile-acllist.c index 28764171b07..6e3fd226d98 100644 --- a/src/plugins/acl/acl-backend-vfile-acllist.c +++ b/src/plugins/acl/acl-backend-vfile-acllist.c @@ -279,7 +279,8 @@ acl_backend_vfile_acllist_try_rebuild(struct acl_backend_vfile *backend) } if (o_stream_nfinish(output) < 0) { - i_error("write(%s) failed: %m", str_c(path)); + i_error("write(%s) failed: %s", str_c(path), + o_stream_get_error(output)); ret = -1; } if (mailbox_list_iter_deinit(&iter) < 0) diff --git a/src/plugins/acl/acl-backend-vfile-update.c b/src/plugins/acl/acl-backend-vfile-update.c index ecd2dea99db..0c2ebadee07 100644 --- a/src/plugins/acl/acl-backend-vfile-update.c +++ b/src/plugins/acl/acl-backend-vfile-update.c @@ -159,7 +159,8 @@ acl_backend_vfile_update_write(struct acl_object *aclobj, } str_free(&str); if (o_stream_nfinish(output) < 0) { - i_error("write(%s) failed: %m", path); + i_error("write(%s) failed: %s", path, + o_stream_get_error(output)); ret = -1; } o_stream_destroy(&output); diff --git a/src/plugins/acl/acl-backend-vfile.c b/src/plugins/acl/acl-backend-vfile.c index 188a41cbaaa..db8048a5df7 100644 --- a/src/plugins/acl/acl-backend-vfile.c +++ b/src/plugins/acl/acl-backend-vfile.c @@ -393,7 +393,8 @@ acl_backend_vfile_read(struct acl_object *aclobj, bool global, const char *path, ret = 0; else { ret = -1; - i_error("read(%s) failed: %m", path); + i_error("read(%s) failed: %s", path, + i_stream_get_error(input)); } } else { if (fstat(fd, &st) < 0) { diff --git a/src/plugins/dict-ldap/Makefile.am b/src/plugins/dict-ldap/Makefile.am new file mode 100644 index 00000000000..a03b9156de3 --- /dev/null +++ b/src/plugins/dict-ldap/Makefile.am @@ -0,0 +1,21 @@ +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-dict \ + -I$(top_srcdir)/src/lib-ldap \ + -I$(top_srcdir)/src/lib-settings + +LIBDICT_LDAP = libdict_ldap.la +libdict_ldap_la_DEPENDENCIES = $(LIBDOVECOT_LDAP) $(LIBDOVECOT_DEPS) +libdict_ldap_la_LDFLAGS = -module -avoid-version +libdict_ldap_la_LIBADD = $(LIBDOVECOT_LDAP) $(LIBDOVECOT) + +module_dictdir = $(moduledir)/dict +module_dict_LTLIBRARIES = \ + $(LIBDICT_LDAP) + +libdict_ldap_la_SOURCES = \ + dict-ldap.c \ + dict-ldap-settings.c + +noinst_HEADERS = \ + dict-ldap-settings.h diff --git a/src/lib-dict/dict-ldap-settings.c b/src/plugins/dict-ldap/dict-ldap-settings.c similarity index 100% rename from src/lib-dict/dict-ldap-settings.c rename to src/plugins/dict-ldap/dict-ldap-settings.c diff --git a/src/lib-dict/dict-ldap-settings.h b/src/plugins/dict-ldap/dict-ldap-settings.h similarity index 100% rename from src/lib-dict/dict-ldap-settings.h rename to src/plugins/dict-ldap/dict-ldap-settings.h diff --git a/src/lib-dict/dict-ldap.c b/src/plugins/dict-ldap/dict-ldap.c similarity index 89% rename from src/lib-dict/dict-ldap.c rename to src/plugins/dict-ldap/dict-ldap.c index 5ea12daa6d1..c241f63367c 100644 --- a/src/lib-dict/dict-ldap.c +++ b/src/plugins/dict-ldap/dict-ldap.c @@ -46,9 +46,6 @@ struct ldap_dict { struct ldap_dict *prev,*next; }; -static -struct ldap_dict *ldap_dict_list; - static void ldap_dict_lookup_async(struct dict *dict, const char *key, dict_lookup_callback_t *callback, void *context); @@ -165,13 +162,14 @@ int dict_ldap_connect(struct ldap_dict *dict, const char **error_r) return ldap_client_init(&set, &dict->client, error_r); } -static -const char* ldap_dict_build_query(struct ldap_dict *dict, const struct dict_ldap_map *map, ARRAY_TYPE(const_string) *values, bool priv) +static void +ldap_dict_build_query(struct ldap_dict *dict, const struct dict_ldap_map *map, + ARRAY_TYPE(const_string) *values, bool priv, + string_t *query_r) { const char *template; ARRAY(struct var_expand_table) exp; struct var_expand_table entry; - string_t *query = t_str_new(64); t_array_init(&exp, 8); entry.key = '\0'; @@ -194,15 +192,13 @@ const char* ldap_dict_build_query(struct ldap_dict *dict, const struct dict_ldap array_append_zero(&exp); - var_expand(query, template, array_idx(&exp, 0)); - - return str_c(query); + var_expand(query_r, template, array_idx(&exp, 0)); } static -int ldap_dict_create(struct dict *dict_driver, const char *uri, - const struct dict_settings *set, - struct dict **dict_r, const char **error_r) +int ldap_dict_init(struct dict *dict_driver, const char *uri, + const struct dict_settings *set, + struct dict **dict_r, const char **error_r) { pool_t pool = pool_alloconly_create("ldap dict", 2048); struct ldap_dict *dict = p_new(pool, struct ldap_dict, 1); @@ -224,31 +220,16 @@ int ldap_dict_create(struct dict *dict_driver, const char *uri, *dict_r = (struct dict*)dict; *error_r = NULL; - - DLLIST_PREPEND(&ldap_dict_list, dict); - return 0; } static -int ldap_dict_init(struct dict *dict_driver, const char *uri, - const struct dict_settings *set, - struct dict **dict_r, const char **error_r) +void ldap_dict_deinit(struct dict *dict) { - /* reuse possible existing entry */ - for(struct ldap_dict *ptr = ldap_dict_list; - ptr != NULL; - ptr = ptr->next) { - if (strcmp(ptr->uri, uri) == 0) { - *dict_r = (struct dict*)ptr; - return 0; - } - } - return ldap_dict_create(dict_driver, uri, set, dict_r, error_r); -} + struct ldap_dict *ctx = (struct ldap_dict *)dict; -static -void ldap_dict_deinit(struct dict *dict ATTR_UNUSED) { + ldap_client_deinit(&ctx->client); + pool_unref(&ctx->pool); } static @@ -259,14 +240,14 @@ int ldap_dict_wait(struct dict *dict) { ctx->prev_ioloop = current_ioloop; ctx->ioloop = io_loop_create(); - ldap_client_switch_ioloop(ctx->client); + dict_switch_ioloop(dict); do { io_loop_run(current_ioloop); } while (ctx->pending > 0); io_loop_set_current(ctx->prev_ioloop); - ldap_client_switch_ioloop(ctx->client); + dict_switch_ioloop(dict); io_loop_set_current(ctx->ioloop); io_loop_destroy(&ctx->ioloop); ctx->prev_ioloop = NULL; @@ -274,11 +255,21 @@ int ldap_dict_wait(struct dict *dict) { return 0; } +static bool ldap_dict_switch_ioloop(struct dict *dict) +{ + struct ldap_dict *ctx = (struct ldap_dict *)dict; + + ldap_client_switch_ioloop(ctx->client); + return ctx->pending > 0; +} + static void ldap_dict_lookup_done(const struct dict_lookup_result *result, void *ctx) { struct dict_lookup_result *res = ctx; - *res = *result; + res->ret = result->ret; + res->value = t_strdup(result->value); + res->error = t_strdup(result->error); } static void @@ -393,6 +384,7 @@ void ldap_dict_lookup_async(struct dict *dict, const char *key, struct ldap_dict *ctx = (struct ldap_dict*)dict; struct dict_ldap_op *op; pool_t oppool = pool_alloconly_create("ldap dict lookup", 64); + string_t *query = str_new(oppool, 64); op = p_new(oppool, struct dict_ldap_op, 1); op->pool = oppool; op->dict = ctx; @@ -414,7 +406,8 @@ void ldap_dict_lookup_async(struct dict *dict, const char *key, memset(&input, 0, sizeof(input)); input.base_dn = map->base_dn; input.scope = map->scope_val; - input.filter = ldap_dict_build_query(ctx, map, &values, strncmp(key, DICT_PATH_PRIVATE, strlen(DICT_PATH_PRIVATE))==0); + ldap_dict_build_query(ctx, map, &values, strncmp(key, DICT_PATH_PRIVATE, strlen(DICT_PATH_PRIVATE))==0, query); + input.filter = str_c(query); input.attributes = attributes; input.timeout_secs = ctx->set->timeout; ctx->pending++; @@ -444,7 +437,8 @@ struct dict dict_driver_ldap = { NULL, /*ldap_unset,*/ NULL, /*ldap_append,*/ NULL, /*ldap_atomic_inc,*/ - ldap_dict_lookup_async + ldap_dict_lookup_async, + ldap_dict_switch_ioloop } }; @@ -454,22 +448,12 @@ void dict_ldap_deinit(void); void dict_ldap_init(struct module *module ATTR_UNUSED) { dict_driver_register(&dict_driver_ldap); - ldap_dict_list = NULL; } void dict_ldap_deinit(void) { + ldap_clients_cleanup(); dict_driver_unregister(&dict_driver_ldap); - /* destroy all server connections */ - struct ldap_dict *ptr = ldap_dict_list; - ldap_dict_list = NULL; - - while(ptr != NULL) { - ldap_client_deinit(&(ptr->client)); - pool_t pool = ptr->pool; - ptr = ptr->next; - pool_unref(&pool); - } } const char *dict_ldap_plugin_dependencies[] = { NULL }; diff --git a/src/plugins/fs-compress/fs-compress.c b/src/plugins/fs-compress/fs-compress.c index 4088c58db7f..81dfb8c66f8 100644 --- a/src/plugins/fs-compress/fs-compress.c +++ b/src/plugins/fs-compress/fs-compress.c @@ -413,6 +413,7 @@ const struct fs fs_class_compress = { fs_compress_delete, fs_compress_iter_init, fs_compress_iter_next, - fs_compress_iter_deinit + fs_compress_iter_deinit, + NULL } }; diff --git a/src/plugins/fts-lucene/fts-backend-lucene.c b/src/plugins/fts-lucene/fts-backend-lucene.c index f098e481b18..3420debe2d6 100644 --- a/src/plugins/fts-lucene/fts-backend-lucene.c +++ b/src/plugins/fts-lucene/fts-backend-lucene.c @@ -178,11 +178,15 @@ fts_backend_lucene_get_last_uid(struct fts_backend *_backend, FTS_LUCENE_USER_CONTEXT(_backend->ns->user); struct fts_index_header hdr; uint32_t set_checksum; + int ret; if (fts_index_get_header(box, &hdr)) { set_checksum = fts_lucene_settings_checksum(&fuser->set); - if (!fts_index_have_compatible_settings(_backend->ns->list, - set_checksum)) { + ret = fts_index_have_compatible_settings(_backend->ns->list, + set_checksum); + if (ret < 0) + return -1; + if (ret == 0) { /* need to rebuild the index */ *last_uid_r = 0; } else { diff --git a/src/plugins/fts-lucene/lucene-wrapper.cc b/src/plugins/fts-lucene/lucene-wrapper.cc index 5d634b03b69..3752f891849 100644 --- a/src/plugins/fts-lucene/lucene-wrapper.cc +++ b/src/plugins/fts-lucene/lucene-wrapper.cc @@ -819,6 +819,30 @@ rescan_next(struct rescan_context *ctx, Document *doc) } } +static void +rescan_clear_unseen_mailbox(struct lucene_index *index, + struct rescan_context *rescan_ctx, + const char *vname, + const struct fts_index_header *hdr) +{ + struct mailbox *box; + struct mailbox_metadata metadata; + + box = mailbox_alloc(index->list, vname, + (enum mailbox_flags)0); + if (mailbox_open(box) == 0 && + mailbox_get_metadata(box, MAILBOX_METADATA_GUID, + &metadata) == 0 && + (rescan_ctx == NULL || + hash_table_lookup(rescan_ctx->seen_mailbox_guids, + metadata.guid) == NULL)) { + /* this mailbox had no records in lucene index. + make sure its last indexed uid is 0 */ + (void)fts_index_set_header(box, hdr); + } + mailbox_free(&box); +} + static void rescan_clear_unseen_mailboxes(struct lucene_index *index, struct rescan_context *rescan_ctx) { @@ -828,30 +852,25 @@ static void rescan_clear_unseen_mailboxes(struct lucene_index *index, MAILBOX_LIST_ITER_RETURN_NO_FLAGS); struct mailbox_list_iterate_context *iter; const struct mailbox_info *info; - struct mailbox *box; - struct mailbox_metadata metadata; struct fts_index_header hdr; + struct mail_namespace *ns = index->list->ns; + const char *vname; memset(&hdr, 0, sizeof(hdr)); hdr.settings_checksum = fts_lucene_settings_checksum(&index->set); iter = mailbox_list_iter_init(index->list, "*", iter_flags); - while ((info = mailbox_list_iter_next(iter)) != NULL) { - box = mailbox_alloc(index->list, info->vname, - (enum mailbox_flags)0); - if (mailbox_open(box) == 0 && - mailbox_get_metadata(box, MAILBOX_METADATA_GUID, - &metadata) == 0 && - (rescan_ctx == NULL || - hash_table_lookup(rescan_ctx->seen_mailbox_guids, - metadata.guid) == NULL)) { - /* this mailbox had no records in lucene index. - make sure its last indexed uid is 0 */ - (void)fts_index_set_header(box, &hdr); - } - mailbox_free(&box); - } + while ((info = mailbox_list_iter_next(iter)) != NULL) + rescan_clear_unseen_mailbox(index, rescan_ctx, info->vname, &hdr); (void)mailbox_list_iter_deinit(&iter); + + if (ns->prefix_len > 0 && + ns->prefix[ns->prefix_len-1] == mail_namespace_get_sep(ns)) { + /* namespace prefix itself isn't returned by the listing */ + vname = t_strndup(index->list->ns->prefix, + index->list->ns->prefix_len-1); + rescan_clear_unseen_mailbox(index, rescan_ctx, vname, &hdr); + } } int lucene_index_rescan(struct lucene_index *index) @@ -1181,6 +1200,9 @@ lucene_add_definite_query(struct lucene_index *index, bool and_args = (flags & FTS_LOOKUP_FLAG_AND_ARGS) != 0; Query *q; + if (arg->no_fts) + return false; + if (arg->match_not && !and_args) { /* FIXME: we could handle this by doing multiple queries.. */ return false; @@ -1247,6 +1269,9 @@ lucene_add_maybe_query(struct lucene_index *index, bool and_args = (flags & FTS_LOOKUP_FLAG_AND_ARGS) != 0; Query *q = NULL; + if (arg->no_fts) + return false; + if (arg->match_not) { /* FIXME: we could handle this by doing multiple queries.. */ return false; diff --git a/src/plugins/fts-solr/fts-backend-solr-old.c b/src/plugins/fts-solr/fts-backend-solr-old.c index c3776441423..5ffbc8ad824 100644 --- a/src/plugins/fts-solr/fts-backend-solr-old.c +++ b/src/plugins/fts-solr/fts-backend-solr-old.c @@ -630,6 +630,8 @@ static int fts_backend_solr_optimize(struct fts_backend *backend ATTR_UNUSED) static bool solr_add_definite_query(string_t *str, struct mail_search_arg *arg) { + if (arg->no_fts) + return FALSE; switch (arg->type) { case SEARCH_TEXT: { if (arg->match_not) diff --git a/src/plugins/fts-solr/fts-backend-solr.c b/src/plugins/fts-solr/fts-backend-solr.c index 3c2da2f73ab..5d0d3ac3902 100644 --- a/src/plugins/fts-solr/fts-backend-solr.c +++ b/src/plugins/fts-solr/fts-backend-solr.c @@ -664,6 +664,8 @@ static void solr_add_str_arg(string_t *str, struct mail_search_arg *arg) static bool solr_add_definite_query(string_t *str, struct mail_search_arg *arg) { + if (arg->no_fts) + return FALSE; switch (arg->type) { case SEARCH_TEXT: { if (arg->match_not) @@ -726,6 +728,8 @@ solr_add_definite_query_args(string_t *str, struct mail_search_arg *arg, static bool solr_add_maybe_query(string_t *str, struct mail_search_arg *arg) { + if (arg->no_fts) + return FALSE; switch (arg->type) { case SEARCH_HEADER: case SEARCH_HEADER_ADDRESS: diff --git a/src/plugins/fts-squat/squat-trie.c b/src/plugins/fts-squat/squat-trie.c index 0861f21b0c8..b3d8fa301b8 100644 --- a/src/plugins/fts-squat/squat-trie.c +++ b/src/plugins/fts-squat/squat-trie.c @@ -1680,7 +1680,8 @@ static int squat_trie_write(struct squat_trie_build_context *ctx) o_stream_nsend(output, &trie->hdr, sizeof(trie->hdr)); } if (o_stream_nfinish(output) < 0) { - i_error("write() to %s failed: %m", path); + i_error("write(%s) failed: %s", path, + o_stream_get_error(output)); ret = -1; } o_stream_destroy(&output); diff --git a/src/plugins/fts-squat/squat-uidlist.c b/src/plugins/fts-squat/squat-uidlist.c index 92bc86fa469..1b7fb1cad19 100644 --- a/src/plugins/fts-squat/squat-uidlist.c +++ b/src/plugins/fts-squat/squat-uidlist.c @@ -861,7 +861,8 @@ int squat_uidlist_build_finish(struct squat_uidlist_build_context *ctx) } if (o_stream_nfinish(ctx->output) < 0) { - i_error("write() to %s failed: %m", ctx->uidlist->path); + i_error("write() to %s failed: %s", ctx->uidlist->path, + o_stream_get_error(ctx->output)); return -1; } return 0; @@ -1064,7 +1065,8 @@ int squat_uidlist_rebuild_finish(struct squat_uidlist_rebuild_context *ctx, if (ctx->uidlist->corrupted) ret = -1; else if (o_stream_nfinish(ctx->output) < 0) { - i_error("write() to %s failed: %m", temp_path); + i_error("write(%s) failed: %s", temp_path, + o_stream_get_error(ctx->output)); ret = -1; } else if (rename(temp_path, ctx->uidlist->path) < 0) { i_error("rename(%s, %s) failed: %m", diff --git a/src/plugins/fts/fts-api.c b/src/plugins/fts/fts-api.c index 66c09c900a7..5eec770bc85 100644 --- a/src/plugins/fts/fts-api.c +++ b/src/plugins/fts/fts-api.c @@ -8,7 +8,6 @@ #include "mail-storage-private.h" #include "mailbox-list-iter.h" #include "mail-search.h" -#include "../virtual/virtual-storage.h" #include "fts-api-private.h" static ARRAY(const struct fts_backend *) backends; @@ -90,7 +89,7 @@ int fts_backend_get_last_uid(struct fts_backend *backend, struct mailbox *box, { struct fts_index_header hdr; - if (strcmp(box->storage->name, VIRTUAL_STORAGE_NAME) == 0) { + if (box->virtual_vfuncs != NULL) { /* virtual mailboxes themselves don't have any indexes, so catch this call here */ if (!fts_index_get_header(box, &hdr)) @@ -228,7 +227,14 @@ int fts_backend_reset_last_uids(struct fts_backend *backend) int fts_backend_rescan(struct fts_backend *backend) { - if (strcmp(backend->ns->storage->name, VIRTUAL_STORAGE_NAME) == 0) { + struct mailbox *box; + bool virtual_storage; + + box = mailbox_alloc(backend->ns->list, "", 0); + virtual_storage = box->virtual_vfuncs != NULL; + mailbox_free(&box); + + if (virtual_storage) { /* just reset the last-uids for a virtual storage. */ return fts_backend_reset_last_uids(backend); } @@ -309,7 +315,9 @@ bool fts_backend_default_can_lookup(struct fts_backend *backend, case SEARCH_HEADER_COMPRESS_LWSP: case SEARCH_BODY: case SEARCH_TEXT: - return TRUE; + if (!args->no_fts) + return TRUE; + break; default: break; } diff --git a/src/plugins/fts/fts-expunge-log.c b/src/plugins/fts/fts-expunge-log.c index 27ce54ee88e..0deb7bed270 100644 --- a/src/plugins/fts/fts-expunge-log.c +++ b/src/plugins/fts/fts-expunge-log.c @@ -424,7 +424,8 @@ fts_expunge_log_read_failure(struct fts_expunge_log_read_ctx *ctx, if (ctx->input->stream_errno != 0) { ctx->failed = TRUE; - i_error("read(%s) failed: %m", ctx->log->path); + i_error("read(%s) failed: %s", ctx->log->path, + i_stream_get_error(ctx->input)); } else { size = i_stream_get_data_size(ctx->input); ctx->corrupted = TRUE; diff --git a/src/plugins/fts/fts-search.c b/src/plugins/fts/fts-search.c index ffbf3d4268b..acdad40dabc 100644 --- a/src/plugins/fts/fts-search.c +++ b/src/plugins/fts/fts-search.c @@ -5,7 +5,6 @@ #include "str.h" #include "seq-range-array.h" #include "mail-search.h" -#include "../virtual/virtual-storage.h" #include "fts-api-private.h" #include "fts-search-args.h" #include "fts-search-serialize.h" @@ -63,7 +62,7 @@ static int fts_search_lookup_level_single(struct fts_search_context *fctx, } static void -level_scores_add_vuids(struct virtual_mailbox *vbox, +level_scores_add_vuids(struct mailbox *box, struct fts_search_level *level, struct fts_result *br) { const struct fts_score_map *scores; @@ -78,8 +77,8 @@ level_scores_add_vuids(struct virtual_mailbox *vbox, t_array_init(&backend_uids, 64); for (i = 0; i < count; i++) seq_range_array_add(&backend_uids, scores[i].uid); - vbox->vfuncs.get_virtual_uid_map(&vbox->box, br->box, - &backend_uids, &vuids_arr); + box->virtual_vfuncs->get_virtual_uid_map(box, br->box, + &backend_uids, &vuids_arr); i_assert(array_count(&vuids_arr) == array_count(&br->scores)); vuids = array_get(&vuids_arr, &count); @@ -110,7 +109,6 @@ multi_add_lookup_result(struct fts_search_context *fctx, struct mail_search_arg *args, struct fts_multi_result *result) { - struct virtual_mailbox *vbox = (struct virtual_mailbox *)fctx->box; ARRAY_TYPE(seq_range) vuids; size_t orig_size; unsigned int i; @@ -132,21 +130,20 @@ multi_add_lookup_result(struct fts_search_context *fctx, array_clear(&vuids); if (array_is_created(&br->definite_uids)) { - vbox->vfuncs.get_virtual_uids(fctx->box, br->box, - &br->definite_uids, - &vuids); + fctx->box->virtual_vfuncs->get_virtual_uids(fctx->box, + br->box, &br->definite_uids, &vuids); } uid_range_to_seqs(fctx, &vuids, &level->definite_seqs); array_clear(&vuids); if (array_is_created(&br->maybe_uids)) { - vbox->vfuncs.get_virtual_uids(fctx->box, br->box, - &br->maybe_uids, &vuids); + fctx->box->virtual_vfuncs->get_virtual_uids(fctx->box, + br->box, &br->maybe_uids, &vuids); } uid_range_to_seqs(fctx, &vuids, &level->maybe_seqs); if (array_is_created(&br->scores)) - level_scores_add_vuids(vbox, level, br); + level_scores_add_vuids(fctx->box, level, br); } return 0; } @@ -157,7 +154,6 @@ static int fts_search_lookup_level_multi(struct fts_search_context *fctx, { enum fts_lookup_flags flags = fctx->flags | (and_args ? FTS_LOOKUP_FLAG_AND_ARGS : 0); - struct virtual_mailbox *vbox = (struct virtual_mailbox *)fctx->box; ARRAY_TYPE(mailboxes) mailboxes_arr, tmp_mailboxes; struct mailbox *const *mailboxes; struct fts_backend *backend; @@ -166,7 +162,8 @@ static int fts_search_lookup_level_multi(struct fts_search_context *fctx, unsigned int i, j, mailbox_count; p_array_init(&mailboxes_arr, fctx->result_pool, 8); - vbox->vfuncs.get_virtual_backend_boxes(fctx->box, &mailboxes_arr, TRUE); + fctx->box->virtual_vfuncs->get_virtual_backend_boxes(fctx->box, + &mailboxes_arr, TRUE); array_sort(&mailboxes_arr, mailbox_cmp_fts_backend); memset(&result, 0, sizeof(result)); diff --git a/src/plugins/fts/fts-storage.c b/src/plugins/fts/fts-storage.c index 4be64b1e660..592ec7aa731 100644 --- a/src/plugins/fts/fts-storage.c +++ b/src/plugins/fts/fts-storage.c @@ -6,10 +6,10 @@ #include "str.h" #include "strescape.h" #include "write-full.h" +#include "wildcard-match.h" #include "mail-search-build.h" #include "mail-storage-private.h" #include "mailbox-list-private.h" -#include "../virtual/virtual-storage.h" #include "fts-api-private.h" #include "fts-tokenizer.h" #include "fts-indexer.h" @@ -40,6 +40,7 @@ struct fts_mailbox_list { struct fts_mailbox { union mailbox_module_context module_ctx; struct fts_backend_update_context *sync_update_ctx; + bool fts_mailbox_excluded; }; struct fts_transaction_context { @@ -155,7 +156,9 @@ static bool fts_want_build_args(const struct mail_search_arg *args) break; case SEARCH_BODY: case SEARCH_TEXT: - return TRUE; + if (!args->no_fts) + return TRUE; + break; default: break; } @@ -208,8 +211,7 @@ fts_mailbox_search_init(struct mailbox_transaction_context *t, fctx->args = args; fctx->result_pool = pool_alloconly_create("fts results", 1024*64); fctx->orig_matches = buffer_create_dynamic(default_pool, 64); - fctx->virtual_mailbox = - strcmp(t->box->storage->name, VIRTUAL_STORAGE_NAME) == 0; + fctx->virtual_mailbox = t->box->virtual_vfuncs != NULL; fctx->enforced = mail_user_plugin_getenv(t->box->storage->user, "fts_enforced") != NULL; @@ -534,8 +536,7 @@ void fts_mail_allocated(struct mail *_mail) fmail = p_new(mail->pool, struct fts_mail, 1); fmail->module_ctx.super = *v; mail->vlast = &fmail->module_ctx.super; - fmail->virtual_mail = - strcmp(_mail->box->storage->name, VIRTUAL_STORAGE_NAME) == 0; + fmail->virtual_mail = _mail->box->virtual_vfuncs != NULL; v->get_special = fts_mail_get_special; v->precache = fts_mail_precache; @@ -630,7 +631,7 @@ fts_transaction_commit(struct mailbox_transaction_context *t, bool autoindex; int ret = 0; - autoindex = ft->mails_saved && + autoindex = ft->mails_saved && !fbox->fts_mailbox_excluded && mail_user_plugin_getenv(box->storage->user, "fts_autoindex") != NULL; @@ -727,6 +728,60 @@ static int fts_copy(struct mail_save_context *ctx, struct mail *mail) return 0; } +static const char *const *fts_exclude_get_patterns(struct mail_user *user) +{ + ARRAY_TYPE(const_string) patterns; + const char *str; + char set_name[21+MAX_INT_STRLEN+1]; + unsigned int i; + + str = mail_user_plugin_getenv(user, "fts_autoindex_exclude"); + if (str == NULL) + return NULL; + + t_array_init(&patterns, 16); + for (i = 2; str != NULL; i++) { + array_append(&patterns, &str, 1); + + if (i_snprintf(set_name, sizeof(set_name), + "fts_autoindex_exclude%u", i) < 0) + i_unreached(); + str = mail_user_plugin_getenv(user, set_name); + } + array_append_zero(&patterns); + return array_idx(&patterns, 0); +} + +static bool fts_autoindex_exclude_match(struct mailbox *box) +{ + const char *const *exclude_list; + unsigned int i; + const struct mailbox_settings *set; + const char *const *special_use; + struct mail_user *user = box->storage->user; + + exclude_list = fts_exclude_get_patterns(user); + if (exclude_list == NULL) + return TRUE; + + set = mailbox_settings_find(mailbox_get_namespace(box), + mailbox_get_vname(box)); + special_use = set == NULL ? NULL : + t_strsplit_spaces(set->special_use, " "); + for (i = 0; exclude_list[i] != NULL; i++) { + if (exclude_list[i][0] == '\\') { + /* \Special-use flag */ + if (str_array_icase_find(special_use, exclude_list[i])) + return TRUE; + } else { + /* mailbox name with wildcards */ + if (wildcard_match(box->name, exclude_list[i])) + return TRUE; + } + } + return FALSE; +} + void fts_mailbox_allocated(struct mailbox *box) { struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list); @@ -739,6 +794,7 @@ void fts_mailbox_allocated(struct mailbox *box) fbox = p_new(box->pool, struct fts_mailbox, 1); fbox->module_ctx.super = *v; box->vlast = &fbox->module_ctx.super; + fbox->fts_mailbox_excluded = fts_autoindex_exclude_match(box); v->get_status = fts_mailbox_get_status; v->search_init = fts_mailbox_search_init; @@ -805,7 +861,7 @@ void fts_mail_namespaces_added(struct mail_namespace *namespaces) const char *name; name = mail_user_plugin_getenv(namespaces->user, "fts"); - if (name == NULL) { + if (name == NULL || name[0] == '\0') { if (namespaces->user->mail_debug) i_debug("fts: No fts setting - plugin disabled"); return; diff --git a/src/plugins/last-login/last-login-plugin.c b/src/plugins/last-login/last-login-plugin.c index ae3f8c3f657..0b59ab2b093 100644 --- a/src/plugins/last-login/last-login-plugin.c +++ b/src/plugins/last-login/last-login-plugin.c @@ -72,7 +72,7 @@ static void last_login_mail_user_created(struct mail_user *user) } dict_value = mail_user_plugin_getenv(user, "last_login_dict"); - if (dict_value == NULL) + if (dict_value == NULL || dict_value[0] == '\0') return; memset(&set, 0, sizeof(set)); diff --git a/src/plugins/lazy-expunge/lazy-expunge-plugin.c b/src/plugins/lazy-expunge/lazy-expunge-plugin.c index 260c6301bf8..1fdbe7fd1ff 100644 --- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c +++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c @@ -27,6 +27,11 @@ #define LAZY_EXPUNGE_MAIL_CONTEXT(obj) \ MODULE_CONTEXT(obj, lazy_expunge_mail_module) +struct lazy_expunge_mail { + union mail_module_context module_ctx; + bool moving; +}; + struct lazy_expunge_mail_user { union mail_user_module_context module_ctx; @@ -52,7 +57,9 @@ struct lazy_expunge_transaction { pool_t pool; HASH_TABLE(const char *, void *) guids; - bool failed; + char *delayed_errstr; + enum mail_error delayed_error; + bool copy_only_last_instance; }; @@ -109,7 +116,8 @@ mailbox_open_or_create(struct mailbox_list *list, struct mailbox *src_box, name = get_dest_vname(list, src_box); - box = mailbox_alloc(list, name, MAILBOX_FLAG_NO_INDEX_FILES); + box = mailbox_alloc(list, name, MAILBOX_FLAG_NO_INDEX_FILES | + MAILBOX_FLAG_SAVEONLY | MAILBOX_FLAG_IGNORE_ACLS); if (mailbox_open(box) == 0) { *error_r = NULL; return box; @@ -124,13 +132,19 @@ mailbox_open_or_create(struct mailbox_list *list, struct mailbox *src_box, } /* try creating and re-opening it. */ - if (mailbox_create(box, NULL, FALSE) < 0 || - mailbox_open(box) < 0) { + if (mailbox_create(box, NULL, FALSE) < 0 && + mailbox_get_last_mail_error(box) != MAIL_ERROR_EXISTS) { *error_r = t_strdup_printf("Failed to create mailbox %s: %s", name, mailbox_get_last_error(box, NULL)); mailbox_free(&box); return NULL; } + if (mailbox_open(box) < 0) { + *error_r = t_strdup_printf("Failed to open created mailbox %s: %s", name, + mailbox_get_last_error(box, NULL)); + mailbox_free(&box); + return NULL; + } return box; } @@ -209,32 +223,75 @@ static int lazy_expunge_mail_is_last_instace(struct mail *_mail) return refcount <= 1 ? 1 : 0; } +static bool lazy_expunge_is_internal_mailbox(struct mailbox *box) +{ + struct mail_namespace *ns = box->list->ns; + struct lazy_expunge_mail_user *luser = + LAZY_EXPUNGE_USER_CONTEXT(ns->user); + struct lazy_expunge_mailbox_list *llist = + LAZY_EXPUNGE_LIST_CONTEXT(box->list); + + if (llist == NULL) { + /* lazy_expunge not enabled at all */ + return FALSE; + } + if (llist->internal_namespace) { + /* lazy-expunge namespace */ + return TRUE; + } + if (luser->lazy_mailbox_vname != NULL && + strcmp(luser->lazy_mailbox_vname, box->vname) == 0) { + /* lazy-expunge mailbox */ + return TRUE; + } + return FALSE; +} + +static void lazy_expunge_set_error(struct lazy_expunge_transaction *lt, + struct mail_storage *storage) +{ + const char *errstr; + enum mail_error error; + + errstr = mail_storage_get_last_error(storage, &error); + if (error == MAIL_ERROR_EXPUNGED) { + /* expunging failed because the mail was already expunged. + we don't want to fail because of that. */ + return; + } + + if (lt->delayed_error != MAIL_ERROR_NONE) + return; + lt->delayed_error = error; + lt->delayed_errstr = i_strdup(errstr); +} + static void lazy_expunge_mail_expunge(struct mail *_mail) { struct mail_namespace *ns = _mail->box->list->ns; struct lazy_expunge_mail_user *luser = LAZY_EXPUNGE_USER_CONTEXT(ns->user); struct mail_private *mail = (struct mail_private *)_mail; - union mail_module_context *mmail = LAZY_EXPUNGE_MAIL_CONTEXT(mail); + struct lazy_expunge_mail *mmail = LAZY_EXPUNGE_MAIL_CONTEXT(mail); struct lazy_expunge_transaction *lt = LAZY_EXPUNGE_CONTEXT(_mail->transaction); - struct lazy_expunge_mailbox_list *llist; - struct mailbox *real_box; struct mail *real_mail; struct mail_save_context *save_ctx; const char *error; + bool moving = mmail->moving; int ret; + /* Clear this in case the mail is used for non-move later on. */ + mmail->moving = FALSE; + /* don't copy the mail if we're expunging from lazy_expunge namespace (even if it's via a virtual mailbox) */ if (mail_get_backend_mail(_mail, &real_mail) < 0) { - lt->failed = TRUE; + lazy_expunge_set_error(lt, _mail->box->storage); return; } - real_box = real_mail->box; - llist = LAZY_EXPUNGE_LIST_CONTEXT(real_box->list); - if (llist != NULL && llist->internal_namespace) { - mmail->super.expunge(_mail); + if (lazy_expunge_is_internal_mailbox(real_mail->box)) { + mmail->module_ctx.super.expunge(_mail); return; } @@ -242,12 +299,14 @@ static void lazy_expunge_mail_expunge(struct mail *_mail) /* we want to copy only the last instance of the mail to lazy_expunge namespace. other instances will be expunged immediately. */ - if ((ret = lazy_expunge_mail_is_last_instace(_mail)) < 0) { - lt->failed = TRUE; + if (moving) + ret = 0; + else if ((ret = lazy_expunge_mail_is_last_instace(_mail)) < 0) { + lazy_expunge_set_error(lt, _mail->box->storage); return; } if (ret == 0) { - mmail->super.expunge(_mail); + mmail->module_ctx.super.expunge(_mail); return; } } @@ -259,14 +318,14 @@ static void lazy_expunge_mail_expunge(struct mail *_mail) mail_storage_set_critical(_mail->box->storage, "lazy_expunge: Couldn't open expunge mailbox: " "%s", error); - lt->failed = TRUE; + lazy_expunge_set_error(lt, _mail->box->storage); return; } if (mailbox_sync(lt->dest_box, 0) < 0) { mail_storage_set_critical(_mail->box->storage, "lazy_expunge: Couldn't sync expunge mailbox"); + lazy_expunge_set_error(lt, lt->dest_box->storage); mailbox_free(<->dest_box); - lt->failed = TRUE; return; } @@ -278,8 +337,20 @@ static void lazy_expunge_mail_expunge(struct mail *_mail) mailbox_save_copy_flags(save_ctx, _mail); save_ctx->data.flags &= ~MAIL_DELETED; if (mailbox_copy(&save_ctx, _mail) < 0 && !_mail->expunged) - lt->failed = TRUE; - mmail->super.expunge(_mail); + lazy_expunge_set_error(lt, lt->dest_box->storage); + mmail->module_ctx.super.expunge(_mail); +} + +static int lazy_expunge_copy(struct mail_save_context *ctx, struct mail *_mail) +{ + struct mail_private *mail = (struct mail_private *)_mail; + union mailbox_module_context *mbox = + LAZY_EXPUNGE_CONTEXT(ctx->transaction->box); + struct lazy_expunge_mail *mmail = LAZY_EXPUNGE_MAIL_CONTEXT(mail); + + if (mmail != NULL) + mmail->moving = ctx->moving; + return mbox->super.copy(ctx, _mail); } static struct mailbox_transaction_context * @@ -310,6 +381,7 @@ static void lazy_expunge_transaction_free(struct lazy_expunge_transaction *lt) hash_table_destroy(<->guids); if (lt->pool != NULL) pool_unref(<->pool); + i_free(lt->delayed_errstr); i_free(lt); } @@ -321,16 +393,24 @@ lazy_expunge_transaction_commit(struct mailbox_transaction_context *ctx, struct lazy_expunge_transaction *lt = LAZY_EXPUNGE_CONTEXT(ctx); int ret; - if (lt->dest_trans != NULL && !lt->failed) { - if (mailbox_transaction_commit(<->dest_trans) < 0) - lt->failed = TRUE; + if (lt->dest_trans != NULL && lt->delayed_error == MAIL_ERROR_NONE) { + if (mailbox_transaction_commit(<->dest_trans) < 0) { + lazy_expunge_set_error(lt, ctx->box->storage); + } } - if (lt->failed) { + if (lt->delayed_error == MAIL_ERROR_NONE) + ret = mbox->super.transaction_commit(ctx, changes_r); + else if (lt->delayed_error != MAIL_ERROR_TEMP) { + mail_storage_set_error(ctx->box->storage, lt->delayed_error, + lt->delayed_errstr); mbox->super.transaction_rollback(ctx); ret = -1; } else { - ret = mbox->super.transaction_commit(ctx, changes_r); + mail_storage_set_critical(ctx->box->storage, + "Lazy-expunge transaction failed: %s", lt->delayed_errstr); + mbox->super.transaction_rollback(ctx); + ret = -1; } lazy_expunge_transaction_free(lt); return ret; @@ -352,17 +432,17 @@ static void lazy_expunge_mail_allocated(struct mail *_mail) LAZY_EXPUNGE_CONTEXT(_mail->transaction); struct mail_private *mail = (struct mail_private *)_mail; struct mail_vfuncs *v = mail->vlast; - union mail_module_context *mmail; + struct lazy_expunge_mail *mmail; if (lt == NULL) return; - mmail = p_new(mail->pool, union mail_module_context, 1); - mmail->super = *v; - mail->vlast = &mmail->super; + mmail = p_new(mail->pool, struct lazy_expunge_mail, 1); + mmail->module_ctx.super = *v; + mail->vlast = &mmail->module_ctx.super; v->expunge = lazy_expunge_mail_expunge; - MODULE_CONTEXT_SET_SELF(mail, lazy_expunge_mail_module, mmail); + MODULE_CONTEXT_SET(mail, lazy_expunge_mail_module, mmail); } static int @@ -391,7 +471,7 @@ static void lazy_expunge_mailbox_allocated(struct mailbox *box) union mailbox_module_context *mbox; struct mailbox_vfuncs *v = box->vlast; - if (llist == NULL) + if (llist == NULL || (box->flags & MAILBOX_FLAG_DELETE_UNSAFE) != 0) return; mbox = p_new(box->pool, union mailbox_module_context, 1); @@ -399,13 +479,18 @@ static void lazy_expunge_mailbox_allocated(struct mailbox *box) box->vlast = &mbox->super; MODULE_CONTEXT_SET_SELF(box, lazy_expunge_mail_storage_module, mbox); - if (!llist->internal_namespace) { + if (!lazy_expunge_is_internal_mailbox(box)) { + v->copy = lazy_expunge_copy; v->transaction_begin = lazy_expunge_transaction_begin; v->transaction_commit = lazy_expunge_transaction_commit; v->transaction_rollback = lazy_expunge_transaction_rollback; v->rename_box = lazy_expunge_mailbox_rename; - } else { + } else if (llist->internal_namespace) { v->rename_box = lazy_expunge_mailbox_rename; + } else { + /* internal mailbox in a non-internal namespace - + don't add any unnecessary restrictions to it. if it's not + wanted, just use the ACL plugin. */ } } @@ -441,22 +526,16 @@ lazy_expunge_mail_namespaces_created(struct mail_namespace *namespaces) return; luser->lazy_ns = mail_namespace_find_prefix(namespaces, luser->env); - if (luser->lazy_ns == NULL) { - /* see if it's set to namespace root itself. in that case we - store all the expunged mails to the namespace root. */ - luser->lazy_ns = mail_namespace_find_prefix_nosep(namespaces, luser->env); - if (luser->lazy_ns != NULL) { - luser->lazy_mailbox_vname = p_strndup(namespaces->user->pool, - luser->lazy_ns->prefix, luser->lazy_ns->prefix_len-1); - } + if (luser->lazy_ns != NULL) { + /* we don't want to override this namespace's expunge operation. */ + llist = LAZY_EXPUNGE_LIST_CONTEXT(luser->lazy_ns->list); + llist->internal_namespace = TRUE; + } else { + /* store the the expunged mails to the specified mailbox. */ + luser->lazy_ns = mail_namespace_find(namespaces, luser->env); + luser->lazy_mailbox_vname = luser->env; } - if (luser->lazy_ns == NULL) - i_fatal("lazy_expunge: Unknown namespace: '%s'", luser->env); mail_namespace_ref(luser->lazy_ns); - - /* we don't want to override this namespace's expunge operation. */ - llist = LAZY_EXPUNGE_LIST_CONTEXT(luser->lazy_ns->list); - llist->internal_namespace = TRUE; } static void lazy_expunge_user_deinit(struct mail_user *user) diff --git a/src/plugins/mailbox-alias/mailbox-alias-plugin.c b/src/plugins/mailbox-alias/mailbox-alias-plugin.c index 6cfe6a801f0..2148b0ac684 100644 --- a/src/plugins/mailbox-alias/mailbox-alias-plugin.c +++ b/src/plugins/mailbox-alias/mailbox-alias-plugin.c @@ -225,7 +225,9 @@ static int mailbox_alias_delete(struct mailbox *box) return -1; } - if (mailbox_is_alias_symlink(box)) { + if ((ret = mailbox_is_alias_symlink(box)) < 0) + return -1; + if (ret > 0) { /* we're deleting an alias mailbox. we'll need to handle this explicitly since box->name points to the original mailbox */ symlink_name = alist->module_ctx.super. @@ -245,12 +247,16 @@ static int mailbox_alias_rename(struct mailbox *src, struct mailbox *dest) struct mailbox_alias_mailbox *abox = MAILBOX_ALIAS_CONTEXT(src); int ret; - if (mailbox_is_alias_symlink(src)) { + if ((ret = mailbox_is_alias_symlink(src)) < 0) + return -1; + else if (ret > 0) { mail_storage_set_error(src->storage, MAIL_ERROR_NOTPOSSIBLE, "Can't rename alias mailboxes"); return -1; } - if (mailbox_is_alias_symlink(dest)) { + if ((ret = mailbox_is_alias_symlink(dest)) < 0) + return -1; + else if (ret > 0) { mail_storage_set_error(src->storage, MAIL_ERROR_NOTPOSSIBLE, "Can't rename to mailbox alias"); return -1; diff --git a/src/plugins/pop3-migration/pop3-migration-plugin.c b/src/plugins/pop3-migration/pop3-migration-plugin.c index 8a2795a4f74..5e0e2b33a39 100644 --- a/src/plugins/pop3-migration/pop3-migration-plugin.c +++ b/src/plugins/pop3-migration/pop3-migration-plugin.c @@ -177,7 +177,7 @@ pop3_header_filter_callback(struct header_filter_istream *input ATTR_UNUSED, int pop3_migration_get_hdr_sha1(uint32_t mail_seq, struct istream *input, uoff_t hdr_size, - unsigned char sha1_r[SHA1_RESULTLEN], + unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN], bool *have_eoh_r) { struct istream *input2; @@ -231,7 +231,7 @@ static unsigned int get_cache_idx(struct mail *mail) } static int -get_hdr_sha1(struct mail *mail, unsigned char sha1_r[SHA1_RESULTLEN]) +get_hdr_sha1(struct mail *mail, unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN]) { struct istream *input; struct message_size hdr_size; @@ -290,7 +290,7 @@ get_hdr_sha1(struct mail *mail, unsigned char sha1_r[SHA1_RESULTLEN]) static bool get_cached_hdr_sha1(struct mail *mail, buffer_t *cache_buf, - unsigned char sha1_r[SHA1_RESULTLEN]) + unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN]) { struct index_mail *imail = (struct index_mail *)mail; diff --git a/src/plugins/pop3-migration/pop3-migration-plugin.h b/src/plugins/pop3-migration/pop3-migration-plugin.h index 510eca00a89..398ffb9c4e2 100644 --- a/src/plugins/pop3-migration/pop3-migration-plugin.h +++ b/src/plugins/pop3-migration/pop3-migration-plugin.h @@ -8,7 +8,7 @@ void pop3_migration_plugin_deinit(void); int pop3_migration_get_hdr_sha1(uint32_t mail_seq, struct istream *input, uoff_t hdr_size, - unsigned char sha1_r[SHA1_RESULTLEN], + unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN], bool *have_eoh_r); #endif diff --git a/src/plugins/push-notification/push-notification-txn-msg.c b/src/plugins/push-notification/push-notification-txn-msg.c index 65e64537ecd..ebc9e2fbf4f 100644 --- a/src/plugins/push-notification/push-notification-txn-msg.c +++ b/src/plugins/push-notification/push-notification-txn-msg.c @@ -45,7 +45,7 @@ push_notification_txn_msg_end(struct push_notification_txn *ptxn, struct push_notification_driver_txn **dtxn; struct seq_range_iter siter; struct mailbox_status status; - uint32_t uid; + uint32_t uid, uid_validity; struct push_notification_txn_msg *value; if (!hash_table_is_created(ptxn->messages)) { @@ -55,20 +55,21 @@ push_notification_txn_msg_end(struct push_notification_txn *ptxn, hiter = hash_table_iterate_init(ptxn->messages); seq_range_array_iter_init(&siter, &changes->saved_uids); + /* uid_validity is only set in changes if message is new. */ + if (changes->uid_validity == 0) { + mailbox_get_open_status(ptxn->mbox, STATUS_UIDVALIDITY, &status); + uid_validity = status.uidvalidity; + } else { + uid_validity = changes->uid_validity; + } + while (hash_table_iterate(hiter, ptxn->messages, &key, &value)) { if (value->uid == 0) { if (seq_range_array_iter_nth(&siter, value->seq, &uid)) { value->uid = uid; } } - - /* uid_validity is only set in changes if message is new. */ - if (changes->uid_validity == 0) { - mailbox_get_open_status(ptxn->mbox, STATUS_UIDVALIDITY, &status); - value->uid_validity = status.uidvalidity; - } else { - value->uid_validity = changes->uid_validity; - } + value->uid_validity = uid_validity; array_foreach_modifiable(&ptxn->drivers, dtxn) { if ((*dtxn)->duser->driver->v.process_msg != NULL) { diff --git a/src/plugins/quota/doveadm-quota.c b/src/plugins/quota/doveadm-quota.c index cbf96d4bb8c..2bdf7dc1465 100644 --- a/src/plugins/quota/doveadm-quota.c +++ b/src/plugins/quota/doveadm-quota.c @@ -101,7 +101,7 @@ cmd_quota_recalc_run(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED, memset(&trans, 0, sizeof(trans)); trans.quota = quser->quota; - trans.recalculate = TRUE; + trans.recalculate = QUOTA_RECALCULATE_FORCED; array_foreach(&quser->quota->roots, root) (void)(*root)->backend.v.update(*root, &trans); diff --git a/src/plugins/quota/quota-count.c b/src/plugins/quota/quota-count.c index 15ef14e95cd..246c46dc589 100644 --- a/src/plugins/quota/quota-count.c +++ b/src/plugins/quota/quota-count.c @@ -1,9 +1,17 @@ /* Copyright (c) 2006-2016 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "ioloop.h" #include "mailbox-list-iter.h" #include "quota-private.h" +struct count_quota_root { + struct quota_root root; + + struct timeval cache_timeval; + uint64_t cached_bytes, cached_count; +}; + struct quota_mailbox_iter { struct quota_root *root; struct mail_namespace *ns; @@ -139,7 +147,7 @@ int quota_count(struct quota_root *root, uint64_t *bytes_r, uint64_t *count_r) { struct quota_mailbox_iter *iter; const struct mailbox_info *info; - int ret = 0; + int ret = 0, ret2; *bytes_r = *count_r = 0; if (root->recounting) @@ -147,18 +155,48 @@ int quota_count(struct quota_root *root, uint64_t *bytes_r, uint64_t *count_r) root->recounting = TRUE; iter = quota_mailbox_iter_begin(root); - while (ret >= 0 && (info = quota_mailbox_iter_next(iter)) != NULL) { - ret = quota_count_mailbox(root, info->ns, info->vname, - bytes_r, count_r); + while ((info = quota_mailbox_iter_next(iter)) != NULL) { + ret2 = quota_count_mailbox(root, info->ns, info->vname, + bytes_r, count_r); + if (ret2 > 0) + ret = 1; + else if (ret2 < 0) { + ret = -1; + break; + } } quota_mailbox_iter_deinit(&iter); root->recounting = FALSE; return ret; } +static int quota_count_cached(struct count_quota_root *root, + uint64_t *bytes_r, uint64_t *count_r) +{ + int ret; + + if (root->cache_timeval.tv_usec == ioloop_timeval.tv_usec && + root->cache_timeval.tv_sec == ioloop_timeval.tv_sec && + ioloop_timeval.tv_sec != 0) { + *bytes_r = root->cached_bytes; + *count_r = root->cached_count; + return 1; + } + ret = quota_count(&root->root, bytes_r, count_r); + if (ret > 0) { + root->cache_timeval = ioloop_timeval; + root->cached_bytes = *bytes_r; + root->cached_count = *count_r; + } + return ret < 0 ? -1 : 0; +} + static struct quota_root *count_quota_alloc(void) { - return i_new(struct quota_root, 1); + struct count_quota_root *root; + + root = i_new(struct count_quota_root, 1); + return &root->root; } static int count_quota_init(struct quota_root *root, const char *args, @@ -168,6 +206,7 @@ static int count_quota_init(struct quota_root *root, const char *args, *error_r = "quota count backend requires quota_vsizes=yes"; return -1; } + root->auto_updating = TRUE; return quota_root_default_init(root, args, error_r); } @@ -186,12 +225,13 @@ count_quota_root_get_resources(struct quota_root *root ATTR_UNUSED) } static int -count_quota_get_resource(struct quota_root *root, +count_quota_get_resource(struct quota_root *_root, const char *name, uint64_t *value_r) { + struct count_quota_root *root = (struct count_quota_root *)_root; uint64_t bytes, count; - if (quota_count(root, &bytes, &count) < 0) + if (quota_count_cached(root, &bytes, &count) < 0) return -1; if (strcmp(name, QUOTA_NAME_STORAGE_BYTES) == 0) @@ -267,7 +307,10 @@ static int count_quota_update(struct quota_root *root, struct quota_transaction_context *ctx) { - if (ctx->recalculate) { + struct count_quota_root *croot = (struct count_quota_root *)root; + + croot->cache_timeval.tv_sec = 0; + if (ctx->recalculate == QUOTA_RECALCULATE_FORCED) { if (quota_count_recalculate(root) < 0) return -1; } diff --git a/src/plugins/quota/quota-dict.c b/src/plugins/quota/quota-dict.c index 4026faa149f..1394a228aa1 100644 --- a/src/plugins/quota/quota-dict.c +++ b/src/plugins/quota/quota-dict.c @@ -217,7 +217,7 @@ dict_quota_update(struct quota_root *_root, struct dict_transaction_context *dt; uint64_t value; - if (ctx->recalculate) { + if (ctx->recalculate != QUOTA_RECALCULATE_DONT) { if (dict_quota_count(root, TRUE, &value) < 0) return -1; } else { diff --git a/src/plugins/quota/quota-dirsize.c b/src/plugins/quota/quota-dirsize.c index 1d214061764..17fe6e5def3 100644 --- a/src/plugins/quota/quota-dirsize.c +++ b/src/plugins/quota/quota-dirsize.c @@ -25,6 +25,13 @@ static struct quota_root *dirsize_quota_alloc(void) return i_new(struct quota_root, 1); } +static int dirsize_quota_init(struct quota_root *root, const char *args, + const char **error_r) +{ + root->auto_updating = TRUE; + return quota_root_default_init(root, args, error_r); +} + static void dirsize_quota_deinit(struct quota_root *_root) { i_free(_root); @@ -210,7 +217,7 @@ struct quota_backend quota_backend_dirsize = { { dirsize_quota_alloc, - NULL, + dirsize_quota_init, dirsize_quota_deinit, NULL, NULL, diff --git a/src/plugins/quota/quota-fs.c b/src/plugins/quota/quota-fs.c index 5b3431a5a86..b3c15be3a88 100644 --- a/src/plugins/quota/quota-fs.c +++ b/src/plugins/quota/quota-fs.c @@ -122,6 +122,7 @@ static int fs_quota_init(struct quota_root *_root, const char *args, return -1; } } + _root->auto_updating = TRUE; return 0; } diff --git a/src/plugins/quota/quota-maildir.c b/src/plugins/quota/quota-maildir.c index df745fc0c10..7c7a23f7a58 100644 --- a/src/plugins/quota/quota-maildir.c +++ b/src/plugins/quota/quota-maildir.c @@ -903,7 +903,7 @@ maildir_quota_update(struct quota_root *_root, we wanted to do. */ } else if (root->fd == -1) (void)maildirsize_recalculate(root); - else if (ctx->recalculate) { + else if (ctx->recalculate != QUOTA_RECALCULATE_DONT) { i_close_fd(&root->fd); (void)maildirsize_recalculate(root); } else if (maildirsize_update(root, ctx->count_used, ctx->bytes_used) < 0) { diff --git a/src/plugins/quota/quota-private.h b/src/plugins/quota/quota-private.h index 5f3e1100ab4..6ac87af181b 100644 --- a/src/plugins/quota/quota-private.h +++ b/src/plugins/quota/quota-private.h @@ -131,14 +131,27 @@ struct quota_root { /* Module-specific contexts. See quota_module_id. */ ARRAY(void) quota_module_contexts; + /* Set to the current quota_over_flag, regardless of whether + it matches quota_over_flag_value mask. */ + const char *quota_over_flag; + /* don't enforce quota when saving */ unsigned int no_enforcing:1; + /* quota is automatically updated. update() should be called but the + bytes/count won't be used. */ + unsigned int auto_updating:1; /* If user has unlimited quota, disable quota tracking */ unsigned int disable_unlimited_tracking:1; /* Set while quota is being recalculated to avoid recursion. */ unsigned int recounting:1; /* Quota root is hidden (to e.g. IMAP GETQUOTAROOT) */ unsigned int hidden:1; + /* Is quota_over_flag* initialized yet? */ + unsigned int quota_over_flag_initialized:1; + /* Is user currently over quota? */ + unsigned int quota_over_flag_status:1; + /* Did we already check quota_over_flag correctness? */ + unsigned int quota_over_flag_checked:1; }; struct quota_transaction_context { @@ -162,11 +175,13 @@ struct quota_transaction_context { uint64_t bytes_over, count_over; struct mail *tmp_mail; + enum quota_recalculate recalculate; unsigned int limits_set:1; unsigned int failed:1; - unsigned int recalculate:1; unsigned int sync_transaction:1; + /* TRUE if all roots have auto_updating=TRUE */ + unsigned int auto_updating:1; }; /* Register storage to all user's quota roots. */ diff --git a/src/plugins/quota/quota-storage.c b/src/plugins/quota/quota-storage.c index 17d56b4ad9e..14270d8df28 100644 --- a/src/plugins/quota/quota-storage.c +++ b/src/plugins/quota/quota-storage.c @@ -31,6 +31,7 @@ struct quota_mailbox { struct quota_transaction_context *expunge_qt; ARRAY(uint32_t) expunge_uids; ARRAY(uoff_t) expunge_sizes; + unsigned int prev_idx; unsigned int recalculate:1; unsigned int sync_transaction_expunge:1; @@ -51,9 +52,15 @@ static void quota_mail_expunge(struct mail *_mail) struct quota_mailbox *qbox = QUOTA_CONTEXT(_mail->box); struct quota_user *quser = QUOTA_USER_CONTEXT(_mail->box->storage->user); union mail_module_context *qmail = QUOTA_MAIL_CONTEXT(mail); + struct quota_transaction_context *qt = QUOTA_CONTEXT(_mail->transaction); uoff_t size; int ret; + if (qt->auto_updating) { + qmail->super.expunge(_mail); + return; + } + /* We need to handle the situation where multiple transactions expunged the mail at the same time. In here we'll just save the message's physical size and do the quota freeing later when the message was @@ -327,7 +334,8 @@ static void quota_mailbox_sync_notify(struct mailbox *box, uint32_t uid, if (qbox->module_ctx.super.sync_notify != NULL) qbox->module_ctx.super.sync_notify(box, uid, sync_type); - if (sync_type != MAILBOX_SYNC_TYPE_EXPUNGE || qbox->recalculate) { + if (sync_type != MAILBOX_SYNC_TYPE_EXPUNGE || qbox->recalculate || + (box->flags & MAILBOX_FLAG_DELETE_UNSAFE) != 0) { if (uid == 0) { /* free the transaction before view syncing begins, otherwise it'll crash. */ @@ -336,24 +344,36 @@ static void quota_mailbox_sync_notify(struct mailbox *box, uint32_t uid, return; } + if (qbox->expunge_qt == NULL) { + qbox->expunge_qt = quota_transaction_begin(box); + qbox->expunge_qt->sync_transaction = + qbox->sync_transaction_expunge; + } + if (qbox->expunge_qt->auto_updating) + return; + /* we're in the middle of syncing the mailbox, so it's a bad idea to try and get the message sizes at this point. Rely on sizes that we saved earlier, or recalculate the whole quota if we don't know the size. */ - if (!array_is_created(&qbox->expunge_uids)) { + if (!array_is_created(&qbox->expunge_uids) || + array_is_empty(&qbox->expunge_uids)) { i = count = 0; } else { uids = array_get(&qbox->expunge_uids, &count); - for (i = 0; i < count; i++) { + for (i = qbox->prev_idx; i < count; i++) { if (uids[i] == uid) break; } - } - - if (qbox->expunge_qt == NULL) { - qbox->expunge_qt = quota_transaction_begin(box); - qbox->expunge_qt->sync_transaction = - qbox->sync_transaction_expunge; + if (i >= count) { + for (i = 0; i < qbox->prev_idx; i++) { + if (uids[i] == uid) + break; + } + if (i == qbox->prev_idx) + i = count; + } + qbox->prev_idx = i; } if (i != count) { @@ -396,7 +416,7 @@ static void quota_mailbox_sync_notify(struct mailbox *box, uint32_t uid, index_mailbox_vsize_hdr_expunge(ibox->vsize_update, uid, size); } else { /* there's no way to get the size. recalculate the quota. */ - quota_recalculate(qbox->expunge_qt); + quota_recalculate(qbox->expunge_qt, QUOTA_RECALCULATE_MISSING_FREES); qbox->recalculate = TRUE; } } @@ -653,5 +673,5 @@ void quota_mail_namespaces_created(struct mail_namespace *namespaces) for (i = 0; i < count; i++) quota_root_set_namespace(roots[i], namespaces); - quota_over_flag_check(namespaces->user, quota); + quota_over_flag_check_startup(quota); } diff --git a/src/plugins/quota/quota.c b/src/plugins/quota/quota.c index d75cee3d288..6d881e1fd33 100644 --- a/src/plugins/quota/quota.c +++ b/src/plugins/quota/quota.c @@ -46,6 +46,7 @@ static const struct quota_backend *quota_backends[] = { static int quota_default_test_alloc(struct quota_transaction_context *ctx, uoff_t size, bool *too_large_r); +static void quota_over_flag_check_root(struct quota_root *root); static const struct quota_backend *quota_backend_find(const char *name) { @@ -655,6 +656,9 @@ const char *quota_root_get_name(struct quota_root *root) const char *const *quota_root_get_resources(struct quota_root *root) { + /* if we haven't checked the quota_over_flag yet, do it now */ + quota_over_flag_check_root(root); + return root->backend.v.get_resources(root); } @@ -748,6 +752,7 @@ int quota_set_resource(struct quota_root *root, const char *name, struct quota_transaction_context *quota_transaction_begin(struct mailbox *box) { struct quota_transaction_context *ctx; + struct quota_root *const *rootp; ctx = i_new(struct quota_transaction_context, 1); ctx->quota = box->list->ns->owner != NULL ? @@ -760,6 +765,12 @@ struct quota_transaction_context *quota_transaction_begin(struct mailbox *box) ctx->bytes_ceil2 = (uint64_t)-1; ctx->count_ceil = (uint64_t)-1; + ctx->auto_updating = TRUE; + array_foreach(&ctx->quota->roots, rootp) { + if (!(*rootp)->auto_updating) + ctx->auto_updating = FALSE; + } + if (box->storage->user->dsyncing) { /* ignore quota for dsync */ ctx->limits_set = TRUE; @@ -944,7 +955,7 @@ int quota_transaction_commit(struct quota_transaction_context **_ctx) if (ctx->failed) ret = -1; else if (ctx->bytes_used != 0 || ctx->count_used != 0 || - ctx->recalculate) T_BEGIN { + ctx->recalculate != QUOTA_RECALCULATE_DONT) T_BEGIN { ARRAY(struct quota_root *) warn_roots; mailbox_name = mailbox_get_vname(ctx->box); @@ -983,34 +994,42 @@ int quota_transaction_commit(struct quota_transaction_context **_ctx) return ret; } -static void -quota_over_flag_check_root(struct mail_user *user, struct quota_root *root) +static void quota_over_flag_init_root(struct quota_root *root) { - const char *name, *flag_mask, *overquota_value, *overquota_script; - const char *const *resources; - unsigned int i; - uint64_t value, limit; - bool overquota_flag, cur_overquota = FALSE; - int ret; + const char *name, *flag_mask; - name = t_strconcat(root->set->set_name, "_over_script", NULL); - overquota_script = mail_user_plugin_getenv(user, name); - if (overquota_script == NULL) + if (root->quota_over_flag_initialized) return; + root->quota_over_flag_initialized = TRUE; /* e.g.: quota_over_flag_value=TRUE or quota_over_flag_value=* */ name = t_strconcat(root->set->set_name, "_over_flag_value", NULL); - flag_mask = mail_user_plugin_getenv(user, name); + flag_mask = mail_user_plugin_getenv(root->quota->user, name); if (flag_mask == NULL) return; /* compare quota_over_flag's value to quota_over_flag_value and save the result. */ name = t_strconcat(root->set->set_name, "_over_flag", NULL); - overquota_value = mail_user_plugin_getenv(user, name); - overquota_flag = overquota_value != NULL && - overquota_value[0] != '\0' && - wildcard_match_icase(overquota_value, flag_mask); + root->quota_over_flag = p_strdup_empty(root->pool, + mail_user_plugin_getenv(root->quota->user, name)); + root->quota_over_flag_status = root->quota_over_flag != NULL && + wildcard_match_icase(root->quota_over_flag, flag_mask); +} + +static void quota_over_flag_check_root(struct quota_root *root) +{ + const char *name, *overquota_script; + const char *const *resources; + unsigned int i; + uint64_t value, limit; + bool cur_overquota = FALSE; + int ret; + + if (root->quota_over_flag_checked) + return; + root->quota_over_flag_checked = TRUE; + quota_over_flag_init_root(root); resources = quota_root_get_resources(root); for (i = 0; resources[i] != NULL; i++) { @@ -1029,26 +1048,35 @@ quota_over_flag_check_root(struct mail_user *user, struct quota_root *root) (unsigned long long)value, (unsigned long long)limit); } - if (ret > 0 && value > limit) + if (ret > 0 && value >= limit) cur_overquota = TRUE; } if (root->quota->set->debug) { i_debug("quota: quota_over_flag=%d(%s) vs currently overquota=%d", - overquota_flag, overquota_value != NULL ? "(null)" : overquota_value, + root->quota_over_flag_status, + root->quota_over_flag == NULL ? "(null)" : root->quota_over_flag, cur_overquota); } - if (cur_overquota != overquota_flag) - quota_warning_execute(root, overquota_script, overquota_value); + if (cur_overquota != root->quota_over_flag_status) { + name = t_strconcat(root->set->set_name, "_over_script", NULL); + overquota_script = mail_user_plugin_getenv(root->quota->user, name); + if (overquota_script != NULL) + quota_warning_execute(root, overquota_script, root->quota_over_flag); + } } -void quota_over_flag_check(struct mail_user *user, struct quota *quota) +void quota_over_flag_check_startup(struct quota *quota) { struct quota_root *const *roots; unsigned int i, count; + const char *name; roots = array_get("a->roots, &count); - for (i = 0; i < count; i++) - quota_over_flag_check_root(user, roots[i]); + for (i = 0; i < count; i++) { + name = t_strconcat(roots[i]->set->set_name, "_over_flag_lazy_check", NULL); + if (mail_user_plugin_getenv(roots[i]->quota->user, name) == NULL) + quota_over_flag_check_root(roots[i]); + } } void quota_transaction_rollback(struct quota_transaction_context **_ctx) @@ -1071,7 +1099,12 @@ int quota_try_alloc(struct quota_transaction_context *ctx, ret = quota_test_alloc(ctx, size, too_large_r); if (ret <= 0) return ret; - + /* with quota_try_alloc() we want to keep track of how many bytes + we've been adding/removing, so disable auto_updating=TRUE + optimization. this of course doesn't work perfectly if + quota_alloc() or quota_free*() was already used within the same + transaction, but that doesn't normally happen. */ + ctx->auto_updating = FALSE; quota_alloc(ctx, mail); return 1; } @@ -1131,6 +1164,8 @@ void quota_alloc(struct quota_transaction_context *ctx, struct mail *mail) { uoff_t size; + if (ctx->auto_updating) + return; if (mail_get_physical_size(mail, &size) == 0) ctx->bytes_used += size; @@ -1142,8 +1177,10 @@ void quota_free(struct quota_transaction_context *ctx, struct mail *mail) { uoff_t size; + if (ctx->auto_updating) + return; if (mail_get_physical_size(mail, &size) < 0) - quota_recalculate(ctx); + quota_recalculate(ctx, QUOTA_RECALCULATE_MISSING_FREES); else quota_free_bytes(ctx, size); } @@ -1155,7 +1192,8 @@ void quota_free_bytes(struct quota_transaction_context *ctx, ctx->count_used--; } -void quota_recalculate(struct quota_transaction_context *ctx) +void quota_recalculate(struct quota_transaction_context *ctx, + enum quota_recalculate recalculate) { - ctx->recalculate = TRUE; + ctx->recalculate = recalculate; } diff --git a/src/plugins/quota/quota.h b/src/plugins/quota/quota.h index 7b13f7bb788..1088f5a9b05 100644 --- a/src/plugins/quota/quota.h +++ b/src/plugins/quota/quota.h @@ -19,6 +19,17 @@ struct quota_root; struct quota_root_iter; struct quota_transaction_context; +enum quota_recalculate { + QUOTA_RECALCULATE_DONT = 0, + /* We may want to recalculate quota because we weren't able to call + quota_free*() correctly for all mails. Quota needs to be + recalculated unless the backend does the quota tracking + internally. */ + QUOTA_RECALCULATE_MISSING_FREES, + /* doveadm quota recalc called - make sure the quota is correct */ + QUOTA_RECALCULATE_FORCED +}; + int quota_user_read_settings(struct mail_user *user, struct quota_settings **set_r, const char **error_r); @@ -81,9 +92,10 @@ void quota_free(struct quota_transaction_context *ctx, struct mail *mail); void quota_free_bytes(struct quota_transaction_context *ctx, uoff_t physical_size); /* Mark the quota to be recalculated */ -void quota_recalculate(struct quota_transaction_context *ctx); +void quota_recalculate(struct quota_transaction_context *ctx, + enum quota_recalculate recalculate); /* Execute quota_over_scripts if needed. */ -void quota_over_flag_check(struct mail_user *user, struct quota *quota); +void quota_over_flag_check_startup(struct quota *quota); #endif diff --git a/src/plugins/stats/mail-stats-fill.c b/src/plugins/stats/mail-stats-fill.c index 7c48f6feab0..3c8ff360e4f 100644 --- a/src/plugins/stats/mail-stats-fill.c +++ b/src/plugins/stats/mail-stats-fill.c @@ -1,6 +1,7 @@ /* Copyright (c) 2011-2016 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "time-util.h" #include "stats-plugin.h" #include "mail-stats.h" @@ -110,12 +111,24 @@ user_trans_stats_get(struct stats_user *suser, struct mail_stats *dest_r) void mail_stats_fill(struct stats_user *suser, struct mail_stats *stats_r) { + static bool getrusage_broken = FALSE; + static struct rusage prev_usage; struct rusage usage; memset(stats_r, 0, sizeof(*stats_r)); /* cputime */ - if (getrusage(RUSAGE_SELF, &usage) < 0) - memset(&usage, 0, sizeof(usage)); + if (getrusage(RUSAGE_SELF, &usage) < 0) { + if (!getrusage_broken) { + i_error("getrusage() failed: %m"); + getrusage_broken = TRUE; + } + usage = prev_usage; + } else if (timeval_diff_usecs(&usage.ru_stime, &prev_usage.ru_stime) < 0) { + /* This seems to be a Linux bug. */ + usage.ru_stime = prev_usage.ru_stime; + } + prev_usage = usage; + stats_r->user_cpu = usage.ru_utime; stats_r->sys_cpu = usage.ru_stime; stats_r->min_faults = usage.ru_minflt; diff --git a/src/plugins/stats/stats-plugin.c b/src/plugins/stats/stats-plugin.c index fa65bb8c344..283817b0bda 100644 --- a/src/plugins/stats/stats-plugin.c +++ b/src/plugins/stats/stats-plugin.c @@ -18,7 +18,7 @@ Must be smaller than MAIL_SESSION_IDLE_TIMEOUT_MSECS in stats server */ #define SESSION_STATS_FORCE_REFRESH_SECS (5*60) #define REFRESH_CHECK_INTERVAL 100 -#define MAIL_STATS_SOCKET_NAME "stats-mail" +#define MAIL_STATS_FIFO_NAME "stats-mail" struct stats_storage { union mail_storage_module_context module_ctx; @@ -379,8 +379,11 @@ static void stats_user_created(struct mail_user *user) } if (global_stats_conn == NULL) { - path = t_strconcat(user->set->base_dir, - "/"MAIL_STATS_SOCKET_NAME, NULL); + path = mail_user_plugin_getenv(user, "stats_notify_path"); + if (path == NULL) + path = MAIL_STATS_FIFO_NAME; + if (path[0] != '/') + path = t_strconcat(user->set->base_dir, "/", path, NULL); global_stats_conn = stats_connection_create(path); } stats_connection_ref(global_stats_conn); diff --git a/src/plugins/virtual/virtual-config.c b/src/plugins/virtual/virtual-config.c index 6fe588e8f4d..270c1881120 100644 --- a/src/plugins/virtual/virtual-config.c +++ b/src/plugins/virtual/virtual-config.c @@ -348,13 +348,11 @@ static int virtual_config_box_metadata_match(struct mailbox *box, imtrans = imap_metadata_transaction_begin(box); ret = imap_metadata_get(imtrans, bbox->metadata_entry, &value); - if (ret < 0) { + if (ret < 0) *error_r = t_strdup(imap_metadata_transaction_get_last_error(imtrans, NULL)); - return -1; - } if (ret > 0) ret = wildcard_match(value.value, bbox->metadata_value) ? 1 : 0; - if (bbox->negative_match) + if (ret >= 0 && bbox->negative_match) ret = ret > 0 ? 0 : 1; (void)imap_metadata_transaction_commit(&imtrans, NULL, NULL); return ret; @@ -445,7 +443,7 @@ static int virtual_config_expand_wildcards(struct virtual_parse_context *ctx, *error_r = mailbox_list_get_last_error(user->namespaces->list, NULL); return -1; } - return 0; + return ret < 0 ? -1 : 0; } static void virtual_config_search_args_dup(struct virtual_mailbox *mbox) diff --git a/src/plugins/virtual/virtual-storage.c b/src/plugins/virtual/virtual-storage.c index 908fe30f81f..5cb46c7add5 100644 --- a/src/plugins/virtual/virtual-storage.c +++ b/src/plugins/virtual/virtual-storage.c @@ -260,7 +260,7 @@ virtual_mailbox_alloc(struct mail_storage *_storage, struct mailbox_list *list, mbox->box.storage = _storage; mbox->box.list = list; mbox->box.mail_vfuncs = &virtual_mail_vfuncs; - mbox->vfuncs = virtual_mailbox_vfuncs; + mbox->box.virtual_vfuncs = &virtual_mailbox_vfuncs; index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX); diff --git a/src/plugins/virtual/virtual-storage.h b/src/plugins/virtual/virtual-storage.h index 697a64c438e..c777091410f 100644 --- a/src/plugins/virtual/virtual-storage.h +++ b/src/plugins/virtual/virtual-storage.h @@ -114,24 +114,6 @@ struct virtual_backend_box { }; ARRAY_DEFINE_TYPE(virtual_backend_box, struct virtual_backend_box *); -struct virtual_mailbox_vfuncs { - /* convert backend UIDs to virtual UIDs. if some backend UID doesn't - exist in mailbox, it's simply ignored */ - void (*get_virtual_uids)(struct mailbox *box, - struct mailbox *backend_mailbox, - const ARRAY_TYPE(seq_range) *backend_uids, - ARRAY_TYPE(seq_range) *virtual_uids_r); - /* like get_virtual_uids(), but if a backend UID doesn't exist, - convert it to 0. */ - void (*get_virtual_uid_map)(struct mailbox *box, - struct mailbox *backend_mailbox, - const ARRAY_TYPE(seq_range) *backend_uids, - ARRAY_TYPE(uint32_t) *virtual_uids_r); - void (*get_virtual_backend_boxes)(struct mailbox *box, - ARRAY_TYPE(mailboxes) *mailboxes, - bool only_with_msgs); -}; - struct virtual_mailbox { struct mailbox box; struct virtual_storage *storage; @@ -161,8 +143,6 @@ struct virtual_mailbox { ARRAY_TYPE(mailbox_virtual_patterns) list_include_patterns; ARRAY_TYPE(mailbox_virtual_patterns) list_exclude_patterns; - struct virtual_mailbox_vfuncs vfuncs; - unsigned int uids_mapped:1; unsigned int sync_initialized:1; unsigned int inconsistent:1; diff --git a/src/plugins/welcome/Makefile.am b/src/plugins/welcome/Makefile.am new file mode 100644 index 00000000000..98968f5dc0b --- /dev/null +++ b/src/plugins/welcome/Makefile.am @@ -0,0 +1,14 @@ +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage + +NOPLUGIN_LDFLAGS = +lib99_welcome_plugin_la_LDFLAGS = -module -avoid-version + +module_LTLIBRARIES = \ + lib99_welcome_plugin.la + +lib99_welcome_plugin_la_SOURCES = \ + welcome-plugin.c diff --git a/src/plugins/welcome/welcome-plugin.c b/src/plugins/welcome/welcome-plugin.c new file mode 100644 index 00000000000..090219c9a2b --- /dev/null +++ b/src/plugins/welcome/welcome-plugin.c @@ -0,0 +1,146 @@ +/* Copyright (c) 2015-2016 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "net.h" +#include "str.h" +#include "eacces-error.h" +#include "write-full.h" +#include "module-context.h" +#include "mail-storage-private.h" + +#define WELCOME_SOCKET_TIMEOUT_SECS 30 + +#define WELCOME_CONTEXT(obj) \ + MODULE_CONTEXT(obj, welcome_storage_module) + +struct welcome_mailbox { + union mailbox_module_context module_ctx; + bool created; +}; + +static MODULE_CONTEXT_DEFINE_INIT(welcome_storage_module, + &mail_storage_module_register); + +static void script_execute(struct mail_user *user, const char *cmd, bool wait) +{ + const char *socket_path, *const *args; + string_t *str; + char buf[1024]; + int fd, ret; + + if (user->mail_debug) + i_debug("welcome: Executing %s (wait=%d)", cmd, wait ? 1 : 0); + + args = t_strsplit_spaces(cmd, " "); + socket_path = args[0]; + args++; + + if (*socket_path != '/') { + socket_path = t_strconcat(user->set->base_dir, "/", + socket_path, NULL); + } + if ((fd = net_connect_unix_with_retries(socket_path, 1000)) < 0) { + if (errno == EACCES) { + i_error("welcome: %s", + eacces_error_get("net_connect_unix", + socket_path)); + } else { + i_error("welcome: net_connect_unix(%s) failed: %m", + socket_path); + } + return; + } + + str = t_str_new(1024); + str_append(str, "VERSION\tscript\t3\t0\n"); + if (!wait) + str_append(str, "noreply\n"); + else + str_append(str, "-\n"); + for (; *args != NULL; args++) { + str_append(str, *args); + str_append_c(str, '\n'); + } + str_append_c(str, '\n'); + + alarm(WELCOME_SOCKET_TIMEOUT_SECS); + net_set_nonblock(fd, FALSE); + if (write_full(fd, str_data(str), str_len(str)) < 0) + i_error("write(%s) failed: %m", socket_path); + else if (wait) { + ret = read(fd, buf, sizeof(buf)); + if (ret < 0) + i_error("welcome: read(%s) failed: %m", socket_path); + else if (ret < 2) + i_error("welcome: %s failed: Only %d bytes read", socket_path, ret); + else if (buf[0] != '+') + i_error("welcome: %s failed: Script returned error", socket_path); + } + if (close(fd) < 0) + i_error("close(%s) failed: %m", socket_path); +} + +static int +welcome_create_box(struct mailbox *box, + const struct mailbox_update *update, bool directory) +{ + struct welcome_mailbox *wbox = WELCOME_CONTEXT(box); + + if (wbox->module_ctx.super.create_box(box, update, directory) < 0) + return -1; + /* the mailbox isn't fully created here yet, so just mark it as created + and wait until open() time to actually run it */ + wbox->created = TRUE; + return 0; +} + +static int welcome_open_box(struct mailbox *box) +{ + struct welcome_mailbox *wbox = WELCOME_CONTEXT(box); + const char *cmd; + + cmd = !wbox->created ? NULL : + mail_user_plugin_getenv(box->storage->user, "welcome_script"); + if (cmd != NULL) { + bool wait = mail_user_plugin_getenv(box->storage->user, + "welcome_wait") != NULL; + script_execute(box->storage->user, cmd, wait); + } + return wbox->module_ctx.super.open(box); +} + +static void welcome_mailbox_allocated(struct mailbox *box) +{ + struct mailbox_vfuncs *v = box->vlast; + struct welcome_mailbox *wbox; + + if (!box->inbox_user) + return; + + wbox = p_new(box->pool, struct welcome_mailbox, 1); + wbox->module_ctx.super = *v; + box->vlast = &wbox->module_ctx.super; + + v->create_box = welcome_create_box; + v->open = welcome_open_box; + MODULE_CONTEXT_SET(box, welcome_storage_module, wbox); +} + +static struct mail_storage_hooks welcome_mail_storage_hooks = { + .mailbox_allocated = welcome_mailbox_allocated +}; + +void welcome_plugin_init(struct module *module); +void welcome_plugin_deinit(void); + +void welcome_plugin_init(struct module *module) +{ + mail_storage_hooks_add(module, &welcome_mail_storage_hooks); +} + +void welcome_plugin_deinit(void) +{ + mail_storage_hooks_remove(&welcome_mail_storage_hooks); +} + +const char *welcome_plugin_version = DOVECOT_ABI_VERSION; diff --git a/src/plugins/zlib/zlib-plugin.c b/src/plugins/zlib/zlib-plugin.c index 02b9a96ce3c..7898b88e745 100644 --- a/src/plugins/zlib/zlib-plugin.c +++ b/src/plugins/zlib/zlib-plugin.c @@ -154,6 +154,25 @@ static int zlib_istream_opened(struct mail *_mail, struct istream **stream) return zmail->module_ctx.super.istream_opened(_mail, stream); } +static void zlib_mail_close(struct mail *_mail) +{ + struct mail_private *mail = (struct mail_private *)_mail; + struct zlib_mail *zmail = ZLIB_MAIL_CONTEXT(mail); + struct zlib_user *zuser = ZLIB_USER_CONTEXT(_mail->box->storage->user); + struct zlib_mail_cache *cache = &zuser->cache; + uoff_t size; + + if (cache->uid == _mail->uid && cache->box == _mail->box) { + /* make sure we have read the entire email into the seekable + stream (which causes the original input stream to be + unrefed). we can't safely keep the original input stream + open after the mail is closed. */ + if (i_stream_get_size(cache->input, TRUE, &size) < 0) + zlib_mail_cache_close(zuser); + } + zmail->module_ctx.super.close(_mail); +} + static void zlib_mail_allocated(struct mail *_mail) { struct zlib_transaction_context *zt = ZLIB_CONTEXT(_mail->transaction); @@ -169,6 +188,7 @@ static void zlib_mail_allocated(struct mail *_mail) mail->vlast = &zmail->module_ctx.super; v->istream_opened = zlib_istream_opened; + v->close = zlib_mail_close; MODULE_CONTEXT_SET(mail, zlib_mail_module, zmail); } diff --git a/src/stats/fifo-input-connection.c b/src/stats/fifo-input-connection.c index 64a623d16e9..4c9e060414e 100644 --- a/src/stats/fifo-input-connection.c +++ b/src/stats/fifo-input-connection.c @@ -1,6 +1,7 @@ /* Copyright (c) 2011-2016 Dovecot authors, see the included COPYING file */ #include "lib.h" +#include "llist.h" #include "strescape.h" #include "istream.h" #include "ostream.h" @@ -15,11 +16,15 @@ #define MAX_INBUF_SIZE (PIPE_BUF*2) struct fifo_input_connection { + struct fifo_input_connection *prev, *next; + int fd; struct istream *input; struct io *io; }; +static struct fifo_input_connection *fifo_conns = NULL; + static int fifo_input_connection_request(const char *const *args, const char **error_r) { @@ -75,6 +80,7 @@ struct fifo_input_connection *fifo_input_connection_create(int fd) conn->fd = fd; conn->input = i_stream_create_fd(fd, MAX_INBUF_SIZE, FALSE); conn->io = io_add(fd, IO_READ, fifo_input_connection_input, conn); + DLLIST_PREPEND(&fifo_conns, conn); return conn; } @@ -84,9 +90,19 @@ void fifo_input_connection_destroy(struct fifo_input_connection **_conn) *_conn = NULL; + DLLIST_REMOVE(&fifo_conns, conn); io_remove(&conn->io); i_stream_destroy(&conn->input); if (close(conn->fd) < 0) i_error("close(conn) failed: %m"); i_free(conn); } + +void fifo_input_connections_destroy_all(void) +{ + while (fifo_conns != NULL) { + struct fifo_input_connection *conn = fifo_conns; + + fifo_input_connection_destroy(&conn); + } +} diff --git a/src/stats/fifo-input-connection.h b/src/stats/fifo-input-connection.h index af6bad644e7..ff394a50603 100644 --- a/src/stats/fifo-input-connection.h +++ b/src/stats/fifo-input-connection.h @@ -4,4 +4,6 @@ struct fifo_input_connection *fifo_input_connection_create(int fd); void fifo_input_connection_destroy(struct fifo_input_connection **conn); +void fifo_input_connections_destroy_all(void); + #endif diff --git a/src/stats/mail-user.c b/src/stats/mail-user.c index 3358483fa76..6e0f7b87c62 100644 --- a/src/stats/mail-user.c +++ b/src/stats/mail-user.c @@ -122,7 +122,7 @@ void mail_user_refresh(struct mail_user *user, int mail_user_add_parse(const char *const *args, const char **error_r) { struct mail_user *user; - struct stats *diff_stats; + struct stats *empty_stats, *diff_stats; buffer_t *buf; const char *service, *error; @@ -141,8 +141,9 @@ int mail_user_add_parse(const char *const *args, const char **error_r) user->name, service); return -1; } + empty_stats = stats_alloc(pool_datastack_create()); diff_stats = stats_alloc(pool_datastack_create()); - if (!stats_import(buf->data, buf->used, user->stats, diff_stats, &error)) { + if (!stats_import(buf->data, buf->used, empty_stats, diff_stats, &error)) { *error_r = t_strdup_printf("ADD-USER %s %s: %s", user->name, service, error); return -1; diff --git a/src/stats/main.c b/src/stats/main.c index e81ce505f44..c731d548501 100644 --- a/src/stats/main.c +++ b/src/stats/main.c @@ -16,20 +16,14 @@ #include "mail-stats.h" #include "client.h" -static struct fifo_input_connection *fifo_input_conn = NULL; static struct module *modules = NULL; static void client_connected(struct master_service_connection *conn) { - if (conn->fifo) { - if (fifo_input_conn != NULL) { - i_error("Received another mail-server connection"); - return; - } - fifo_input_conn = fifo_input_connection_create(conn->fd); - } else { + if (conn->fifo) + (void)fifo_input_connection_create(conn->fd); + else (void)client_create(conn->fd); - } master_service_client_connection_accept(conn); } @@ -85,6 +79,7 @@ int main(int argc, char *argv[]) master_service_run(master_service, client_connected); clients_destroy_all(); + fifo_input_connections_destroy_all(); mail_commands_deinit(); mail_sessions_deinit(); mail_users_deinit(); @@ -92,9 +87,6 @@ int main(int argc, char *argv[]) mail_ips_deinit(); mail_global_deinit(); - if (fifo_input_conn != NULL) - fifo_input_connection_destroy(&fifo_input_conn); - module_dir_unload(&modules); i_assert(global_used_memory == 0); master_service_deinit(&master_service);