From ee73d9657b65e4ff2111e5d2af7107a4eab0c361 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 13 Apr 2016 13:47:51 +0300
Subject: [PATCH 001/450] doveconf: Improved the warning message about global
 setting not overriding a filter

---
 src/config/config-parser.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/config/config-parser.c b/src/config/config-parser.c
index 169a3425cb6..2a5009af7a8 100644
--- a/src/config/config-parser.c
+++ b/src/config/config-parser.c
@@ -849,9 +849,10 @@ config_parser_check_warnings(struct config_parser_context *ctx, const char *key)
 		   filters, log a warning. */
 		if (first_pos == NULL)
 			return;
-		i_warning("%s line %u: Global setting %s won't change the setting inside an earlier filter at %s",
+		i_warning("%s line %u: Global setting %s won't change the setting inside an earlier filter at %s "
+			  "(if this is intentional, avoid this warning by moving the global setting before %s)",
 			  ctx->cur_input->path, ctx->cur_input->linenum,
-			  key, first_pos);
+			  key, first_pos, first_pos);
 		return;
 	}
 	if (first_pos != NULL)

From c6f4485a09e4b4ff480a30328679f6b47b39da67 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 13 Apr 2016 17:17:53 +0300
Subject: [PATCH 002/450] lib-dict: Don't build dict-ldap if --with-ldap=no

---
 src/lib-dict/Makefile.am | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/lib-dict/Makefile.am b/src/lib-dict/Makefile.am
index ddd4a130ce8..bf893b0f886 100644
--- a/src/lib-dict/Makefile.am
+++ b/src/lib-dict/Makefile.am
@@ -36,11 +36,15 @@ nodist_libdict_backend_a_SOURCES = \
 	dict-drivers-register.c
 
 NOPLUGIN_LDFLAGS =
+
+if HAVE_LDAP
+LIBDICT_LDAP = libdict_ldap.la
+endif
 libdict_ldap_la_LDFLAGS = -module -avoid-version $(LIBDOVECOT_LDAP)
 
 module_dictdir = $(moduledir)/dict
 module_dict_LTLIBRARIES = \
-	libdict_ldap.la
+	$(LIBDICT_LDAP)
 
 libdict_ldap_la_SOURCES = \
 	dict-ldap.c \

From b025075e4874f6e9ce98883a88554f02f02bdfc5 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 13 Apr 2016 17:19:52 +0300
Subject: [PATCH 003/450] configure: Fixed building lib-ldap / dict-ldap
 --with-ldap=plugin

---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 8d7445fefe1..bf0217ea25c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2529,7 +2529,7 @@ else
   LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/liblogin.la'
   LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/liblda.la'
 fi
-if test $want_ldap = yes; then
+if test $want_ldap != no; then
   LIBDOVECOT_LDAP='$(top_builddir)/src/lib-ldap/libdovecot-ldap.la'
 else
   LIBDOVECOT_LDAP=''

From 6da64a8fd41363c7667e1adab03b58e05e92a611 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@rename-it.nl>
Date: Wed, 13 Apr 2016 23:12:00 +0200
Subject: [PATCH 004/450] lib-http: server: Prevent aborting finished or
 already aborted requests again.

---
 src/lib-http/http-server-request.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/lib-http/http-server-request.c b/src/lib-http/http-server-request.c
index 3d0671ef236..34d7a3744ce 100644
--- a/src/lib-http/http-server-request.c
+++ b/src/lib-http/http-server-request.c
@@ -128,6 +128,9 @@ void http_server_request_abort(struct http_server_request **_req,
 	struct http_server_request *req = *_req;
 	struct http_server_connection *conn = req->conn;
 
+	if (req->state >= HTTP_SERVER_REQUEST_STATE_FINISHED)
+		return;
+
 	http_server_request_debug(req, "Abort");
 
 	req->conn = NULL;

From 14e2d75759a311ee6baf3b0416380fc7b3ed2448 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 15 Apr 2016 15:01:20 +0300
Subject: [PATCH 005/450] lib-stats: Handle better write() to stats process
 failing with EAGAIN

It only means that the stats process is too busy and the FIFO is filled up.
Retrying the write later should work. We also don't want to log too much about
the same warning, so do it only once per 30 seconds.
---
 src/lib-stats/stats-connection.c | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/src/lib-stats/stats-connection.c b/src/lib-stats/stats-connection.c
index 389ed861c48..740acad7df5 100644
--- a/src/lib-stats/stats-connection.c
+++ b/src/lib-stats/stats-connection.c
@@ -1,6 +1,7 @@
 /* Copyright (c) 2011-2016 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
+#include "ioloop.h"
 #include "str.h"
 #include "master-service.h"
 #include "stats-connection.h"
@@ -8,6 +9,8 @@
 #include <unistd.h>
 #include <fcntl.h>
 
+#define STATS_EAGAIN_WARN_INTERVAL_SECS 30
+
 struct stats_connection {
 	int refcount;
 
@@ -15,6 +18,7 @@ struct stats_connection {
 	char *path;
 
 	bool open_failed;
+	time_t next_warning_timestamp;
 };
 
 static bool stats_connection_open(struct stats_connection *conn)
@@ -89,7 +93,17 @@ void stats_connection_send(struct stats_connection *conn, const string_t *str)
 	}
 
 	ret = write(conn->fd, str_data(str), str_len(str));
-	if (ret != (ssize_t)str_len(str)) {
+	if (ret == (ssize_t)str_len(str)) {
+		/* success */
+	} else if (ret < 0 && errno == EAGAIN) {
+		/* stats process is busy */
+		if (ioloop_time > conn->next_warning_timestamp) {
+			i_warning("write(%s) failed: %m (stats process is busy)", conn->path);
+			conn->next_warning_timestamp = ioloop_time +
+				STATS_EAGAIN_WARN_INTERVAL_SECS;
+		}
+	} else {
+		/* error - reconnect */
 		if (ret < 0) {
 			/* don't log EPIPE errors. they can happen when
 			   Dovecot is stopped. */

From 40e57479542a648353db2211580d350c6d9c5432 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 15 Apr 2016 14:51:42 +0300
Subject: [PATCH 006/450] stats plugin: stats_notify_path can now specify path
 to the stats-mail FIFO

---
 src/plugins/stats/stats-plugin.c | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/plugins/stats/stats-plugin.c b/src/plugins/stats/stats-plugin.c
index fa65bb8c344..283817b0bda 100644
--- a/src/plugins/stats/stats-plugin.c
+++ b/src/plugins/stats/stats-plugin.c
@@ -18,7 +18,7 @@
    Must be smaller than MAIL_SESSION_IDLE_TIMEOUT_MSECS in stats server */
 #define SESSION_STATS_FORCE_REFRESH_SECS (5*60)
 #define REFRESH_CHECK_INTERVAL 100
-#define MAIL_STATS_SOCKET_NAME "stats-mail"
+#define MAIL_STATS_FIFO_NAME "stats-mail"
 
 struct stats_storage {
 	union mail_storage_module_context module_ctx;
@@ -379,8 +379,11 @@ static void stats_user_created(struct mail_user *user)
 	}
 
 	if (global_stats_conn == NULL) {
-		path = t_strconcat(user->set->base_dir,
-				   "/"MAIL_STATS_SOCKET_NAME, NULL);
+		path = mail_user_plugin_getenv(user, "stats_notify_path");
+		if (path == NULL)
+			path = MAIL_STATS_FIFO_NAME;
+		if (path[0] != '/')
+			path = t_strconcat(user->set->base_dir, "/", path, NULL);
 		global_stats_conn = stats_connection_create(path);
 	}
 	stats_connection_ref(global_stats_conn);

From 6d85cb1eccb2c2ea2ba7b724b31748152e550c5d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 19 Apr 2016 16:59:48 +0300
Subject: [PATCH 007/450] lib-storage: Added
 MAIL_STORAGE_CLASS_FLAG_NO_LIST_DELETES and MAILBOX_LIST_FLAG_NO_DELETES

---
 src/lib-storage/list/mailbox-list-index-backend.c | 6 ++++--
 src/lib-storage/mail-storage-private.h            | 5 ++++-
 src/lib-storage/mail-storage.c                    | 2 ++
 src/lib-storage/mailbox-list.h                    | 4 +++-
 4 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/lib-storage/list/mailbox-list-index-backend.c b/src/lib-storage/list/mailbox-list-index-backend.c
index da40d0aa3c4..4cef1bcf693 100644
--- a/src/lib-storage/list/mailbox-list-index-backend.c
+++ b/src/lib-storage/list/mailbox-list-index-backend.c
@@ -526,7 +526,8 @@ index_list_delete_mailbox(struct mailbox_list *_list, const char *name)
 	if (ret <= 0)
 		return ret;
 
-	if ((_list->flags & MAILBOX_LIST_FLAG_NO_MAIL_FILES) != 0) {
+	if ((_list->flags & (MAILBOX_LIST_FLAG_NO_MAIL_FILES |
+			     MAILBOX_LIST_FLAG_NO_DELETES)) != 0) {
 		ret = 0;
 	} else if ((_list->flags & MAILBOX_LIST_FLAG_MAILBOX_FILES) != 0) {
 		ret = mailbox_list_delete_mailbox_file(_list, name, path);
@@ -535,7 +536,8 @@ index_list_delete_mailbox(struct mailbox_list *_list, const char *name)
 							       path, TRUE);
 	}
 
-	if (ret == 0 || (_list->props & MAILBOX_LIST_PROP_AUTOCREATE_DIRS) != 0)
+	if ((ret == 0 || (_list->props & MAILBOX_LIST_PROP_AUTOCREATE_DIRS) != 0) &&
+	    (_list->flags & MAILBOX_LIST_FLAG_NO_DELETES) == 0)
 		index_list_delete_finish(_list, name);
 	if (ret == 0) {
 		if (index_list_delete_entry(list, name, TRUE) < 0)
diff --git a/src/lib-storage/mail-storage-private.h b/src/lib-storage/mail-storage-private.h
index d1f1a8250c8..6a2b53756e5 100644
--- a/src/lib-storage/mail-storage-private.h
+++ b/src/lib-storage/mail-storage-private.h
@@ -78,7 +78,10 @@ enum mail_storage_class_flags {
 	MAIL_STORAGE_CLASS_FLAG_BINARY_DATA	= 0x100,
 	/* Message GUIDs can only be 128bit (always set
 	   mailbox_status.have_only_guid128) */
-	MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUID128 = 0x200
+	MAIL_STORAGE_CLASS_FLAG_HAVE_MAIL_GUID128 = 0x200,
+	/* Storage deletes all files internally - mailbox list's
+	   delete_mailbox() shouldn't delete anything itself. */
+	MAIL_STORAGE_CLASS_FLAG_NO_LIST_DELETES	= 0x400
 };
 
 struct mail_binary_cache {
diff --git a/src/lib-storage/mail-storage.c b/src/lib-storage/mail-storage.c
index 331b867d82f..454df24a9bc 100644
--- a/src/lib-storage/mail-storage.c
+++ b/src/lib-storage/mail-storage.c
@@ -360,6 +360,8 @@ int mail_storage_create_full(struct mail_namespace *ns, const char *driver,
 			list_flags |= MAILBOX_LIST_FLAG_MAILBOX_FILES;
 		if ((storage_class->class_flags & MAIL_STORAGE_CLASS_FLAG_NO_ROOT) != 0)
 			list_flags |= MAILBOX_LIST_FLAG_NO_MAIL_FILES;
+		if ((storage_class->class_flags & MAIL_STORAGE_CLASS_FLAG_NO_LIST_DELETES) != 0)
+			list_flags |= MAILBOX_LIST_FLAG_NO_DELETES;
 		if (mailbox_list_create(list_set.layout, ns, &list_set,
 					list_flags, &list, error_r) < 0) {
 			*error_r = t_strdup_printf("Mailbox list driver %s: %s",
diff --git a/src/lib-storage/mailbox-list.h b/src/lib-storage/mailbox-list.h
index 8f5998c3ebe..ae953837bcb 100644
--- a/src/lib-storage/mailbox-list.h
+++ b/src/lib-storage/mailbox-list.h
@@ -34,7 +34,9 @@ enum mailbox_list_flags {
 	   mailbox list to it. */
 	MAILBOX_LIST_FLAG_SECONDARY		= 0x02,
 	/* There are no mail files, only index and/or control files. */
-	MAILBOX_LIST_FLAG_NO_MAIL_FILES		= 0x04
+	MAILBOX_LIST_FLAG_NO_MAIL_FILES		= 0x04,
+	/* LAYOUT=index: Don't delete any files in delete_mailbox(). */
+	MAILBOX_LIST_FLAG_NO_DELETES		= 0x08
 };
 
 enum mailbox_info_flags {

From d1e4867000401287b8c61110e9c4b3c26e86fe66 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 19 Apr 2016 11:55:00 +0300
Subject: [PATCH 008/450] auth: Added passdb/userdb { auth_verbose } setting.

If this is explicitly set to yes or no, it overrides the global auth_verbose
setting. However, auth_debug=yes overrides all of the auth_verbose settings.
---
 src/auth/auth-request.c  | 21 +++++++++++++++++++--
 src/auth/auth-settings.c | 10 ++++++++--
 src/auth/auth-settings.h |  2 ++
 3 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/src/auth/auth-request.c b/src/auth/auth-request.c
index 24242d9d131..90095824b7a 100644
--- a/src/auth/auth-request.c
+++ b/src/auth/auth-request.c
@@ -2185,8 +2185,25 @@ void auth_request_log_info(struct auth_request *auth_request,
 {
 	va_list va;
 
-	if (!auth_request->set->verbose)
-		return;
+	if (auth_request->set->debug) {
+		/* auth_debug=yes overrides auth_verbose settings */
+	} else {
+		const char *db_auth_verbose = auth_request->userdb_lookup ?
+			auth_request->userdb->set->auth_verbose :
+			auth_request->passdb->set->auth_verbose;
+		switch (db_auth_verbose[0]) {
+		case 'y':
+			break;
+		case 'n':
+			return;
+		case 'd':
+			if (!auth_request->set->verbose)
+				return;
+			break;
+		default:
+			i_unreached();
+		}
+	}
 
 	va_start(va, format);
 	T_BEGIN {
diff --git a/src/auth/auth-settings.c b/src/auth/auth-settings.c
index 7eed12f5dbe..e5f6c76131c 100644
--- a/src/auth/auth-settings.c
+++ b/src/auth/auth-settings.c
@@ -121,6 +121,7 @@ static const struct setting_define auth_passdb_setting_defines[] = {
 	DEF(SET_BOOL, deny),
 	DEF(SET_BOOL, pass),
 	DEF(SET_BOOL, master),
+	DEF(SET_ENUM, auth_verbose),
 
 	SETTING_DEFINE_LIST_END
 };
@@ -139,7 +140,8 @@ static const struct auth_passdb_settings auth_passdb_default_settings = {
 
 	.deny = FALSE,
 	.pass = FALSE,
-	.master = FALSE
+	.master = FALSE,
+	.auth_verbose = "default:yes:no"
 };
 
 const struct setting_parser_info auth_passdb_setting_parser_info = {
@@ -171,6 +173,8 @@ static const struct setting_define auth_userdb_setting_defines[] = {
 	DEF(SET_ENUM, result_failure),
 	DEF(SET_ENUM, result_internalfail),
 
+	DEF(SET_ENUM, auth_verbose),
+
 	SETTING_DEFINE_LIST_END
 };
 
@@ -185,7 +189,9 @@ static const struct auth_userdb_settings auth_userdb_default_settings = {
 	.skip = "never:found:notfound",
 	.result_success = "return-ok:return:return-fail:continue:continue-ok:continue-fail",
 	.result_failure = "continue:return:return-ok:return-fail:continue-ok:continue-fail",
-	.result_internalfail = "continue:return:return-ok:return-fail:continue-ok:continue-fail"
+	.result_internalfail = "continue:return:return-ok:return-fail:continue-ok:continue-fail",
+
+	.auth_verbose = "default:yes:no"
 };
 
 const struct setting_parser_info auth_userdb_setting_parser_info = {
diff --git a/src/auth/auth-settings.h b/src/auth/auth-settings.h
index c39f05151bb..295420fa0af 100644
--- a/src/auth/auth-settings.h
+++ b/src/auth/auth-settings.h
@@ -18,6 +18,7 @@ struct auth_passdb_settings {
 	bool deny;
 	bool pass; /* deprecated, use result_success=continue instead */
 	bool master;
+	const char *auth_verbose;
 };
 
 struct auth_userdb_settings {
@@ -31,6 +32,7 @@ struct auth_userdb_settings {
 	const char *result_success;
 	const char *result_failure;
 	const char *result_internalfail;
+	const char *auth_verbose;
 };
 
 struct auth_settings {

From 882807103c79f850484cae830964289b40dc6071 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 18 Apr 2016 16:40:49 +0300
Subject: [PATCH 009/450] lib: Implement utc_mktime() with timegm() if it
 exists.

It should be more efficient than repeatedly calling gmtime() many times.
---
 configure.ac         |  2 +-
 src/lib/utc-mktime.c | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index bf0217ea25c..84297442e5a 100644
--- a/configure.ac
+++ b/configure.ac
@@ -456,7 +456,7 @@ AC_CHECK_FUNCS(fcntl flock lockf inet_aton sigaction getpagesize madvise \
 	       getmntinfo setpriority quotactl getmntent kqueue kevent \
 	       backtrace_symbols walkcontext dirfd clearenv \
 	       malloc_usable_size glob fallocate posix_fadvise \
-	       getpeereid getpeerucred inotify_init)
+	       getpeereid getpeerucred inotify_init timegm)
 
 AC_CHECK_TYPES([struct sockpeercred],,,[
 #include <sys/types.h>
diff --git a/src/lib/utc-mktime.c b/src/lib/utc-mktime.c
index 89522579bc3..566cb44dc26 100644
--- a/src/lib/utc-mktime.c
+++ b/src/lib/utc-mktime.c
@@ -20,6 +20,18 @@ static int tm_cmp(const struct tm *tm1, const struct tm *tm2)
 	return tm1->tm_sec - tm2->tm_sec;
 }
 
+#ifdef HAVE_TIMEGM
+time_t utc_mktime(const struct tm *tm)
+{
+	struct tm mod_tm = *tm;
+	time_t t;
+
+	t = timegm(&mod_tm);
+	if (tm_cmp(tm, &mod_tm) != 0)
+		return (time_t)-1;
+	return t;
+}
+#else
 time_t utc_mktime(const struct tm *tm)
 {
 	const struct tm *try_tm;
@@ -51,3 +63,4 @@ time_t utc_mktime(const struct tm *tm)
 
 	return (time_t)-1;
 }
+#endif

From b1c7419b9238488ba56a47dc8acdc7cb85d65b12 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 14 Apr 2016 19:04:28 +0300
Subject: [PATCH 010/450] lazy-expunge: Allow lazy_expunge setting to point to
 a mailbox in any namespace.

This way we can use an existing namespace without having to create a new one
just for a single lazy_expunge mailbox.
---
 src/plugins/lazy-expunge/lazy-expunge-plugin.c | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/src/plugins/lazy-expunge/lazy-expunge-plugin.c b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
index 260c6301bf8..59530e080c6 100644
--- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c
+++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
@@ -442,16 +442,10 @@ lazy_expunge_mail_namespaces_created(struct mail_namespace *namespaces)
 
 	luser->lazy_ns = mail_namespace_find_prefix(namespaces, luser->env);
 	if (luser->lazy_ns == NULL) {
-		/* see if it's set to namespace root itself. in that case we
-		   store all the expunged mails to the namespace root. */
-		luser->lazy_ns = mail_namespace_find_prefix_nosep(namespaces, luser->env);
-		if (luser->lazy_ns != NULL) {
-			luser->lazy_mailbox_vname = p_strndup(namespaces->user->pool,
-				luser->lazy_ns->prefix, luser->lazy_ns->prefix_len-1);
-		}
+		/* store the the expunged mails to the specified mailbox. */
+		luser->lazy_ns = mail_namespace_find(namespaces, luser->env);
+		luser->lazy_mailbox_vname = luser->env;
 	}
-	if (luser->lazy_ns == NULL)
-		i_fatal("lazy_expunge: Unknown namespace: '%s'", luser->env);
 	mail_namespace_ref(luser->lazy_ns);
 
 	/* we don't want to override this namespace's expunge operation. */

From 31349fd1e282b84d98ca535cce71dc695fb194e0 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 20 Apr 2016 02:47:13 +0300
Subject: [PATCH 011/450] zlib plugin: Make sure we don't keep mail istream
 referenced after mail is closed.

---
 src/plugins/zlib/zlib-plugin.c | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/src/plugins/zlib/zlib-plugin.c b/src/plugins/zlib/zlib-plugin.c
index 02b9a96ce3c..b121565b4d7 100644
--- a/src/plugins/zlib/zlib-plugin.c
+++ b/src/plugins/zlib/zlib-plugin.c
@@ -154,6 +154,25 @@ static int zlib_istream_opened(struct mail *_mail, struct istream **stream)
 	return zmail->module_ctx.super.istream_opened(_mail, stream);
 }
 
+static void zlib_mail_close(struct mail *_mail)
+{
+	struct mail_private *mail = (struct mail_private *)_mail;
+	struct zlib_mail *zmail = ZLIB_MAIL_CONTEXT(mail);
+	struct zlib_user *zuser = ZLIB_USER_CONTEXT(_mail->box->storage->user);
+	struct zlib_mail_cache *cache = &zuser->cache;
+	uoff_t size;
+
+	if (cache->uid == _mail->uid && cache->box == _mail->box) {
+		/* make sure we have read the entire email into the seekable
+		   stream (which causes the original input stream to be
+		   unrefed). we can't safely keep the original input stream
+		   open after the mail is closed. */
+		if (i_stream_get_size(cache->input, TRUE, &size) < 0)
+			zlib_mail_cache_close(zuser);
+	}
+	return zmail->module_ctx.super.close(_mail);
+}
+
 static void zlib_mail_allocated(struct mail *_mail)
 {
 	struct zlib_transaction_context *zt = ZLIB_CONTEXT(_mail->transaction);
@@ -169,6 +188,7 @@ static void zlib_mail_allocated(struct mail *_mail)
 	mail->vlast = &zmail->module_ctx.super;
 
 	v->istream_opened = zlib_istream_opened;
+	v->close = zlib_mail_close;
 	MODULE_CONTEXT_SET(mail, zlib_mail_module, zmail);
 }
 

From 06bed4f40e87a0597ef6128dd2c19881df6e9b5c Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 20 Apr 2016 00:29:49 +0300
Subject: [PATCH 012/450] lib-index: Fixed calling mail_cache_open_and_verify()
 on an already open cache.

This was done at least by index_index_rebuild_init().

Either the currently open cache->fd was leaked, or if the cache file open()
failed we left the cache in an inconsistent state where cache->fd == -1, but
cache->hdr != NULL, so it caused MAIL_CACHE_IS_UNUSABLE() to also be TRUE. This
could have ended up in an assert:

Panic: file mail-index-lock.c: line 31 (mail_index_lock_fd): assertion failed: (MAIL_INDEX_IS_IN_MEMORY(index))
---
 src/lib-index/mail-cache.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/lib-index/mail-cache.c b/src/lib-index/mail-cache.c
index 34c05c5d84b..53b45d58e32 100644
--- a/src/lib-index/mail-cache.c
+++ b/src/lib-index/mail-cache.c
@@ -487,6 +487,8 @@ int mail_cache_open_and_verify(struct mail_cache *cache)
 {
 	int ret;
 
+	if (cache->opened)
+		return 0;
 	ret = mail_cache_try_open(cache);
 	if (ret > 0)
 		ret = mail_cache_header_fields_read(cache);

From 61c30c7cd60c9c95b138175a5c8ab702b472399b Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 20 Apr 2016 02:23:31 +0300
Subject: [PATCH 013/450] lib-index: If opening a cache file fails, try again
 later.

The previous code would never retry opening the cache file within the same
session.
---
 src/lib-index/mail-cache-compress.c | 1 +
 src/lib-index/mail-cache.c          | 8 +++++++-
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/src/lib-index/mail-cache-compress.c b/src/lib-index/mail-cache-compress.c
index 1e82fe2ffa5..cfc33de57c8 100644
--- a/src/lib-index/mail-cache-compress.c
+++ b/src/lib-index/mail-cache-compress.c
@@ -363,6 +363,7 @@ mail_cache_compress_write(struct mail_cache *cache,
 	}
 
 	mail_cache_file_close(cache);
+	cache->opened = TRUE;
 	cache->fd = fd;
 	cache->st_ino = st.st_ino;
 	cache->st_dev = st.st_dev;
diff --git a/src/lib-index/mail-cache.c b/src/lib-index/mail-cache.c
index 53b45d58e32..b5b4876b0ed 100644
--- a/src/lib-index/mail-cache.c
+++ b/src/lib-index/mail-cache.c
@@ -77,6 +77,7 @@ void mail_cache_file_close(struct mail_cache *cache)
 			mail_cache_set_syscall_error(cache, "close()");
 		cache->fd = -1;
 	}
+	cache->opened = FALSE;
 }
 
 static void mail_cache_init_file_cache(struct mail_cache *cache)
@@ -101,14 +102,17 @@ static int mail_cache_try_open(struct mail_cache *cache)
 {
 	const void *data;
 
+	i_assert(!cache->opened);
 	cache->opened = TRUE;
 
 	if (MAIL_INDEX_IS_IN_MEMORY(cache->index))
 		return 0;
 
+	i_assert(cache->fd == -1);
 	cache->fd = nfs_safe_open(cache->filepath,
 				  cache->index->readonly ? O_RDONLY : O_RDWR);
 	if (cache->fd == -1) {
+		mail_cache_file_close(cache);
 		if (errno == ENOENT) {
 			cache->need_compress_file_seq = 0;
 			return 0;
@@ -120,8 +124,10 @@ static int mail_cache_try_open(struct mail_cache *cache)
 
 	mail_cache_init_file_cache(cache);
 
-	if (mail_cache_map(cache, 0, 0, &data) < 0)
+	if (mail_cache_map(cache, 0, 0, &data) < 0) {
+		mail_cache_file_close(cache);
 		return -1;
+	}
 	return 1;
 }
 

From a6a14c762bca9397cccfcd4ef6c841ea5b1d0dcd Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 20 Apr 2016 17:10:17 +0300
Subject: [PATCH 014/450] lazy-expunge: Fixed using a mailbox (instead of
 namespace) as lazy_expunge destination.

The initial implementation in f2d7ae020bda762f78e8e639a69fb129230cbb7d was
completely broken.
---
 .../lazy-expunge/lazy-expunge-plugin.c        | 48 ++++++++++++++-----
 1 file changed, 36 insertions(+), 12 deletions(-)

diff --git a/src/plugins/lazy-expunge/lazy-expunge-plugin.c b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
index 59530e080c6..8a357d79ce7 100644
--- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c
+++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
@@ -209,6 +209,30 @@ static int lazy_expunge_mail_is_last_instace(struct mail *_mail)
 	return refcount <= 1 ? 1 : 0;
 }
 
+static bool lazy_expunge_is_internal_mailbox(struct mailbox *box)
+{
+	struct mail_namespace *ns = box->list->ns;
+	struct lazy_expunge_mail_user *luser =
+		LAZY_EXPUNGE_USER_CONTEXT(ns->user);
+	struct lazy_expunge_mailbox_list *llist =
+		LAZY_EXPUNGE_LIST_CONTEXT(box->list);
+
+	if (llist == NULL) {
+		/* lazy_expunge not enabled at all */
+		return FALSE;
+	}
+	if (llist->internal_namespace) {
+		/* lazy-expunge namespace */
+		return TRUE;
+	}
+	if (luser->lazy_mailbox_vname != NULL &&
+	    strcmp(luser->lazy_mailbox_vname, box->vname) == 0) {
+		/* lazy-expunge mailbox */
+		return TRUE;
+	}
+	return FALSE;
+}
+
 static void lazy_expunge_mail_expunge(struct mail *_mail)
 {
 	struct mail_namespace *ns = _mail->box->list->ns;
@@ -218,8 +242,6 @@ static void lazy_expunge_mail_expunge(struct mail *_mail)
 	union mail_module_context *mmail = LAZY_EXPUNGE_MAIL_CONTEXT(mail);
 	struct lazy_expunge_transaction *lt =
 		LAZY_EXPUNGE_CONTEXT(_mail->transaction);
-	struct lazy_expunge_mailbox_list *llist;
-	struct mailbox *real_box;
 	struct mail *real_mail;
 	struct mail_save_context *save_ctx;
 	const char *error;
@@ -231,9 +253,7 @@ static void lazy_expunge_mail_expunge(struct mail *_mail)
 		lt->failed = TRUE;
 		return;
 	}
-	real_box = real_mail->box;
-	llist = LAZY_EXPUNGE_LIST_CONTEXT(real_box->list);
-	if (llist != NULL && llist->internal_namespace) {
+	if (lazy_expunge_is_internal_mailbox(real_mail->box)) {
 		mmail->super.expunge(_mail);
 		return;
 	}
@@ -399,13 +419,17 @@ static void lazy_expunge_mailbox_allocated(struct mailbox *box)
 	box->vlast = &mbox->super;
 	MODULE_CONTEXT_SET_SELF(box, lazy_expunge_mail_storage_module, mbox);
 
-	if (!llist->internal_namespace) {
+	if (!lazy_expunge_is_internal_mailbox(box)) {
 		v->transaction_begin = lazy_expunge_transaction_begin;
 		v->transaction_commit = lazy_expunge_transaction_commit;
 		v->transaction_rollback = lazy_expunge_transaction_rollback;
 		v->rename_box = lazy_expunge_mailbox_rename;
-	} else {
+	} else if (llist->internal_namespace) {
 		v->rename_box = lazy_expunge_mailbox_rename;
+	} else {
+		/* internal mailbox in a non-internal namespace -
+		   don't add any unnecessary restrictions to it. if it's not
+		   wanted, just use the ACL plugin. */
 	}
 }
 
@@ -441,16 +465,16 @@ lazy_expunge_mail_namespaces_created(struct mail_namespace *namespaces)
 		return;
 
 	luser->lazy_ns = mail_namespace_find_prefix(namespaces, luser->env);
-	if (luser->lazy_ns == NULL) {
+	if (luser->lazy_ns != NULL) {
+		/* we don't want to override this namespace's expunge operation. */
+		llist = LAZY_EXPUNGE_LIST_CONTEXT(luser->lazy_ns->list);
+		llist->internal_namespace = TRUE;
+	} else {
 		/* store the the expunged mails to the specified mailbox. */
 		luser->lazy_ns = mail_namespace_find(namespaces, luser->env);
 		luser->lazy_mailbox_vname = luser->env;
 	}
 	mail_namespace_ref(luser->lazy_ns);
-
-	/* we don't want to override this namespace's expunge operation. */
-	llist = LAZY_EXPUNGE_LIST_CONTEXT(luser->lazy_ns->list);
-	llist->internal_namespace = TRUE;
 }
 
 static void lazy_expunge_user_deinit(struct mail_user *user)

From 7ce6777f7e99cedf81714d21f95ec53c233e253e Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 20 Apr 2016 19:34:05 +0300
Subject: [PATCH 015/450] zlib: Compiling fix to a1630a3b3 - don't return a
 void function

---
 src/plugins/zlib/zlib-plugin.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plugins/zlib/zlib-plugin.c b/src/plugins/zlib/zlib-plugin.c
index b121565b4d7..7898b88e745 100644
--- a/src/plugins/zlib/zlib-plugin.c
+++ b/src/plugins/zlib/zlib-plugin.c
@@ -170,7 +170,7 @@ static void zlib_mail_close(struct mail *_mail)
 		if (i_stream_get_size(cache->input, TRUE, &size) < 0)
 			zlib_mail_cache_close(zuser);
 	}
-	return zmail->module_ctx.super.close(_mail);
+	zmail->module_ctx.super.close(_mail);
 }
 
 static void zlib_mail_allocated(struct mail *_mail)

From 2a9968850577cb9a4c0248960405b6cd2c63ae80 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Fri, 15 Apr 2016 23:59:24 +0200
Subject: [PATCH 016/450] lib-http: client: Implemented means to set request
 payload buffer rather than an input stream.

This is not purely a convenience function: there have been bugs caused by allocating a data input stream from a datastack buffer.
With this function, the buffer is copied to the request pool, so that it is durably allocated while the request exists.
This prevents futher mishaps. The server already has an equivalent function for its response object.
---
 src/lib-http/http-client-request.c | 17 +++++++++++++++++
 src/lib-http/http-client.h         |  2 ++
 2 files changed, 19 insertions(+)

diff --git a/src/lib-http/http-client-request.c b/src/lib-http/http-client-request.c
index ce1a00d45ac..cde2db97c64 100644
--- a/src/lib-http/http-client-request.c
+++ b/src/lib-http/http-client-request.c
@@ -356,6 +356,23 @@ void http_client_request_set_payload(struct http_client_request *req,
 		req->payload_sync = TRUE;
 }
 
+void http_client_request_set_payload_data(struct http_client_request *req,
+				     const unsigned char *data, size_t size)
+{
+	struct istream *input;
+	unsigned char *payload_data;
+
+	if (size == 0)
+		return;
+
+	payload_data = p_malloc(req->pool, size);
+	memcpy(payload_data, data, size);
+	input = i_stream_create_from_data(payload_data, size);
+
+	http_client_request_set_payload(req, input, FALSE);
+	i_stream_unref(&input);
+}
+
 void http_client_request_set_timeout_msecs(struct http_client_request *req,
 	unsigned int msecs)
 {
diff --git a/src/lib-http/http-client.h b/src/lib-http/http-client.h
index 9f59c58d04d..ddc221e538a 100644
--- a/src/lib-http/http-client.h
+++ b/src/lib-http/http-client.h
@@ -204,6 +204,8 @@ void http_client_request_set_date(struct http_client_request *req,
 
 void http_client_request_set_payload(struct http_client_request *req,
 				     struct istream *input, bool sync);
+void http_client_request_set_payload_data(struct http_client_request *req,
+				     const unsigned char *data, size_t size);
 
 void http_client_request_set_timeout_msecs(struct http_client_request *req,
 	unsigned int msecs);

From b9e1531c80039ee75239d31cdad70d430d255974 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 21 Apr 2016 21:41:23 +0300
Subject: [PATCH 017/450] lib-ssl-iostream: Fixed reporting errors returned by
 OpenSSL.

We were always logging all errors as "Stacked error" and then returning
"Unknown error".
---
 src/lib-ssl-iostream/iostream-openssl-common.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-ssl-iostream/iostream-openssl-common.c b/src/lib-ssl-iostream/iostream-openssl-common.c
index cbdbc4407b3..572689763d7 100644
--- a/src/lib-ssl-iostream/iostream-openssl-common.c
+++ b/src/lib-ssl-iostream/iostream-openssl-common.c
@@ -192,7 +192,7 @@ const char *openssl_iostream_error(void)
 	while ((err = ERR_get_error_line_data(NULL, NULL, &data, &flags)) != 0) {
 		if (ERR_GET_REASON(err) == ERR_R_MALLOC_FAILURE)
 			i_fatal_status(FATAL_OUTOFMEM, "OpenSSL malloc() failed");
-		if (ERR_peek_error() != 0)
+		if (ERR_peek_error() == 0)
 			break;
 		i_error("SSL: Stacked error: %s",
 			ssl_err2str(err, data, flags));

From ceefe5b61a235361f863de3114d72c70bbd07a5e Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 17 Apr 2016 21:15:30 +0300
Subject: [PATCH 018/450] cassandra: Added support for user and password
 settings.

---
 src/lib-sql/driver-cassandra.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/lib-sql/driver-cassandra.c b/src/lib-sql/driver-cassandra.c
index e36edf2a818..231e72328b4 100644
--- a/src/lib-sql/driver-cassandra.c
+++ b/src/lib-sql/driver-cassandra.c
@@ -46,7 +46,7 @@ struct cassandra_callback {
 struct cassandra_db {
 	struct sql_db api;
 
-	char *hosts, *keyspace;
+	char *hosts, *keyspace, *user, *password;
 	CassConsistency read_consistency, write_consistency, delete_consistency;
 	CassConsistency read_fallback_consistency, write_fallback_consistency, delete_fallback_consistency;
 	CassLogLevel log_level;
@@ -395,6 +395,12 @@ static void driver_cassandra_parse_connect_string(struct cassandra_db *db,
 			   strcmp(key, "keyspace") == 0) {
 			i_free(db->keyspace);
 			db->keyspace = i_strdup(value);
+		} else if (strcmp(key, "user") == 0) {
+			i_free(db->user);
+			db->user = i_strdup(value);
+		} else if (strcmp(key, "password") == 0) {
+			i_free(db->password);
+			db->password = i_strdup(value);
 		} else if (strcmp(key, "read_consistency") == 0) {
 			if (consistency_parse(value, &db->read_consistency) < 0)
 				i_fatal("cassandra: Unknown read_consistency: %s", value);
@@ -460,6 +466,8 @@ static struct sql_db *driver_cassandra_init_v(const char *connect_string)
 	cass_cluster_set_connect_timeout(db->cluster, SQL_CONNECT_TIMEOUT_SECS * 1000);
 	cass_cluster_set_request_timeout(db->cluster, SQL_QUERY_TIMEOUT_SECS * 1000);
 	cass_cluster_set_contact_points(db->cluster, db->hosts);
+	if (db->user != NULL && db->password != NULL)
+		cass_cluster_set_credentials(db->cluster, db->user, db->password);
 	if (db->port != 0)
 		cass_cluster_set_port(db->cluster, db->port);
 	if (db->protocol_version != 0)
@@ -487,6 +495,8 @@ static void driver_cassandra_deinit_v(struct sql_db *_db)
 	i_free(db->hosts);
 	i_free(db->error);
 	i_free(db->keyspace);
+	i_free(db->user);
+	i_free(db->password);
 	array_free(&_db->module_contexts);
 	i_free(db);
 }

From 81672c783e6df3ed9f03c8589acb0afd9ec32305 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 22 Apr 2016 20:10:34 +0300
Subject: [PATCH 019/450] lib: Improved istream-timeout error message.

We're supposed to check that timeout isn't triggered after a long-running
code, but it's not perfect. So provide the actual timing information we
saw instead of the expected timeout.
---
 src/lib/istream-timeout.c | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/lib/istream-timeout.c b/src/lib/istream-timeout.c
index b3dd4719020..d5ab1c8fc03 100644
--- a/src/lib/istream-timeout.c
+++ b/src/lib/istream-timeout.c
@@ -38,7 +38,7 @@ static void i_stream_timeout_switch_ioloop(struct istream_private *stream)
 
 static void i_stream_timeout(struct timeout_istream *tstream)
 {
-	unsigned int msecs;
+	unsigned int over_msecs;
 	int diff;
 
 	if (tstream->update_timestamp) {
@@ -59,13 +59,14 @@ static void i_stream_timeout(struct timeout_istream *tstream)
 					  i_stream_timeout, tstream);
 		return;
 	}
+	over_msecs = diff - tstream->timeout_msecs;
 
-	msecs = tstream->timeout_msecs % 1000;
 	io_stream_set_error(&tstream->istream.iostream,
-			    "Read timeout in %u%s s after %"PRIuUOFF_T" bytes",
-			    tstream->timeout_msecs/1000,
-			    msecs == 0 ? "" : t_strdup_printf(".%u", msecs),
-			    tstream->istream.istream.v_offset);
+			    "Read timeout in %u.%03u s after %"PRIuUOFF_T" bytes%s",
+			    diff/1000, diff%1000,
+			    tstream->istream.istream.v_offset,
+			    over_msecs < 1000 ? "" : t_strdup_printf(
+			    	" (requested timeout in %u s)", tstream->timeout_secs));
 	tstream->istream.istream.stream_errno = ETIMEDOUT;
 
 	i_stream_set_input_pending(tstream->istream.parent, TRUE);

From 58e4e5b370d52300f351559b3ba8ebce5c42b350 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 22 Apr 2016 20:31:02 +0300
Subject: [PATCH 020/450] lib: Fix to previous istream-timeout commit

The one commit I didn't bother putting through all the automated testing
stages and I of course forgot to even compile after the last change :(
---
 src/lib/istream-timeout.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/istream-timeout.c b/src/lib/istream-timeout.c
index d5ab1c8fc03..50b23ffc4ac 100644
--- a/src/lib/istream-timeout.c
+++ b/src/lib/istream-timeout.c
@@ -66,7 +66,7 @@ static void i_stream_timeout(struct timeout_istream *tstream)
 			    diff/1000, diff%1000,
 			    tstream->istream.istream.v_offset,
 			    over_msecs < 1000 ? "" : t_strdup_printf(
-			    	" (requested timeout in %u s)", tstream->timeout_secs));
+			    	" (requested timeout in %u ms)", tstream->timeout_msecs));
 	tstream->istream.istream.stream_errno = ETIMEDOUT;
 
 	i_stream_set_input_pending(tstream->istream.parent, TRUE);

From 11c704a3d321ee6ae4b6e50c2f22d63845178f10 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 22 Apr 2016 00:21:12 +0300
Subject: [PATCH 021/450] imapc: Added support for imapc_features=modseq

If the remote server supports CONDSTORE or QRESYNC extensions we'll use the
remote's MODSEQ and HIGHESTMODSEQ counts.

There are some situations where the HIGHESTMODSEQ isn't updated exactly
correctly on an open mailbox, so this feature shouldn't be fully relied on.
It was primarily implemented for dsync+imapc support - both for preserving
modseqs and also for HIGHESTMODSEQ lookups.
---
 src/lib-storage/index/imapc/imapc-mail.c     | 22 +++++++-
 src/lib-storage/index/imapc/imapc-mailbox.c  | 34 +++++++++++-
 src/lib-storage/index/imapc/imapc-settings.c |  1 +
 src/lib-storage/index/imapc/imapc-settings.h |  3 +-
 src/lib-storage/index/imapc/imapc-storage.c  | 58 ++++++++++++++++++--
 src/lib-storage/index/imapc/imapc-storage.h  |  3 +
 src/lib-storage/index/imapc/imapc-sync.c     | 15 ++++-
 7 files changed, 127 insertions(+), 9 deletions(-)

diff --git a/src/lib-storage/index/imapc/imapc-mail.c b/src/lib-storage/index/imapc/imapc-mail.c
index 1923a8f66a2..45be11e3a0e 100644
--- a/src/lib-storage/index/imapc/imapc-mail.c
+++ b/src/lib-storage/index/imapc/imapc-mail.c
@@ -96,6 +96,26 @@ static int imapc_mail_failed(struct mail *mail, const char *field)
 	return fix_broken_mail ? 0 : -1;
 }
 
+static uint64_t imapc_mail_get_modseq(struct mail *_mail)
+{
+	struct imapc_mailbox *mbox = (struct imapc_mailbox *)_mail->box;
+	struct imapc_msgmap *msgmap;
+	const uint64_t *modseqs;
+	unsigned int count;
+	uint32_t rseq;
+
+	if (!imapc_storage_has_modseqs(mbox->storage))
+		return index_mail_get_modseq(_mail);
+
+	msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
+	if (imapc_msgmap_uid_to_rseq(msgmap, _mail->uid, &rseq)) {
+		modseqs = array_get(&mbox->rseq_modseqs, &count);
+		if (rseq <= count)
+			return modseqs[rseq-1];
+	}
+	return 1; /* unknown modseq */
+}
+
 static int imapc_mail_get_received_date(struct mail *_mail, time_t *date_r)
 {
 	struct index_mail *mail = (struct index_mail *)_mail;
@@ -561,7 +581,7 @@ struct mail_vfuncs imapc_mail_vfuncs = {
 	index_mail_get_flags,
 	index_mail_get_keywords,
 	index_mail_get_keyword_indexes,
-	index_mail_get_modseq,
+	imapc_mail_get_modseq,
 	index_mail_get_pvt_modseq,
 	index_mail_get_parts,
 	index_mail_get_date,
diff --git a/src/lib-storage/index/imapc/imapc-mailbox.c b/src/lib-storage/index/imapc/imapc-mailbox.c
index c3e12d17021..349339db429 100644
--- a/src/lib-storage/index/imapc/imapc-mailbox.c
+++ b/src/lib-storage/index/imapc/imapc-mailbox.c
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "ioloop.h"
+#include "mail-index-modseq.h"
 #include "imap-arg.h"
 #include "imap-seqset.h"
 #include "imap-util.h"
@@ -282,11 +283,12 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
 	uint32_t lseq, rseq = reply->num;
 	struct imapc_fetch_request *const *fetch_requestp;
 	struct imapc_mail *const *mailp;
-	const struct imap_arg *list, *flags_list;
+	const struct imap_arg *list, *flags_list, *modseq_list;
 	const char *atom, *guid = NULL;
 	const struct mail_index_record *rec = NULL;
 	enum mail_flags flags;
 	uint32_t fetch_uid, uid;
+	uint64_t modseq = 0;
 	unsigned int i, j;
 	ARRAY_TYPE(const_string) keywords = ARRAY_INIT;
 	bool seen_flags = FALSE, have_labels = FALSE;
@@ -319,6 +321,15 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
 					array_append(&keywords, &atom, 1);
 				}
 			}
+		} else if (strcasecmp(atom, "MODSEQ") == 0 &&
+			   imapc_storage_has_modseqs(mbox->storage)) {
+			/* (modseq-number) */
+			if (!imap_arg_get_list(&list[i+1], &modseq_list))
+				return;
+			if (!imap_arg_get_atom(&modseq_list[0], &atom) ||
+			    str_to_uint64(atom, &modseq) < 0 ||
+			    modseq_list[1].type != IMAP_ARG_EOL)
+				return;
 		} else if (strcasecmp(atom, "X-GM-MSGID") == 0 &&
 			   !mbox->initial_sync_done) {
 			if (imap_arg_get_atom(&list[i+1], &atom))
@@ -414,6 +425,11 @@ static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
 		}
 		mail_index_keywords_unref(&kw);
 	}
+	if (modseq != 0) {
+		if (mail_index_modseq_lookup(mbox->delayed_sync_view, lseq) < modseq)
+			mail_index_update_modseq(mbox->delayed_sync_trans, lseq, modseq);
+		array_idx_set(&mbox->rseq_modseqs, rseq-1, &modseq);
+	}
 	if (guid != NULL) {
 		struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(&mbox->box);
 		const enum index_cache_field guid_cache_idx =
@@ -454,6 +470,7 @@ static void imapc_untagged_expunge(const struct imapc_untagged_reply *reply,
 	}
 	uid = imapc_msgmap_rseq_to_uid(msgmap, rseq);
 	imapc_msgmap_expunge(msgmap, rseq);
+	array_delete(&mbox->rseq_modseqs, rseq-1, 1);
 
 	imapc_mailbox_init_delayed_trans(mbox);
 	if (mail_index_lookup_seq(mbox->sync_view, uid, &lseq))
@@ -578,6 +595,19 @@ imapc_resp_text_uidnext(const struct imapc_untagged_reply *reply,
 	mbox->sync_uid_next = uid_next;
 }
 
+static void
+imapc_resp_text_highestmodseq(const struct imapc_untagged_reply *reply,
+			      struct imapc_mailbox *mbox)
+{
+	uint64_t highestmodseq;
+
+	if (mbox == NULL ||
+	    str_to_uint64(reply->resp_text_value, &highestmodseq) < 0)
+		return;
+
+	mbox->sync_highestmodseq = highestmodseq;
+}
+
 static void
 imapc_resp_text_permanentflags(const struct imapc_untagged_reply *reply,
 			       struct imapc_mailbox *mbox)
@@ -646,6 +676,8 @@ void imapc_mailbox_register_callbacks(struct imapc_mailbox *mbox)
 					 imapc_resp_text_uidvalidity);
 	imapc_mailbox_register_resp_text(mbox, "UIDNEXT",
 					 imapc_resp_text_uidnext);
+	imapc_mailbox_register_resp_text(mbox, "HIGHESTMODSEQ",
+					 imapc_resp_text_highestmodseq);
 	imapc_mailbox_register_resp_text(mbox, "PERMANENTFLAGS",
 					 imapc_resp_text_permanentflags);
 }
diff --git a/src/lib-storage/index/imapc/imapc-settings.c b/src/lib-storage/index/imapc/imapc-settings.c
index 6325aa8349f..2bbf274fc74 100644
--- a/src/lib-storage/index/imapc/imapc-settings.c
+++ b/src/lib-storage/index/imapc/imapc-settings.c
@@ -93,6 +93,7 @@ static const struct imapc_feature_list imapc_feature_list[] = {
 	{ "proxyauth", IMAPC_FEATURE_PROXYAUTH },
 	{ "fetch-msn-workarounds", IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS },
 	{ "fetch-fix-broken-mails", IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS },
+	{ "modseq", IMAPC_FEATURE_MODSEQ },
 	{ NULL, 0 }
 };
 
diff --git a/src/lib-storage/index/imapc/imapc-settings.h b/src/lib-storage/index/imapc/imapc-settings.h
index bb1dec1ae73..fa8be08492f 100644
--- a/src/lib-storage/index/imapc/imapc-settings.h
+++ b/src/lib-storage/index/imapc/imapc-settings.h
@@ -14,7 +14,8 @@ enum imapc_features {
 	IMAPC_FEATURE_NO_EXAMINE		= 0x40,
 	IMAPC_FEATURE_PROXYAUTH			= 0x80,
 	IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS	= 0x100,
-	IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS	= 0x200
+	IMAPC_FEATURE_FETCH_FIX_BROKEN_MAILS	= 0x200,
+	IMAPC_FEATURE_MODSEQ			= 0x400
 };
 /* </settings checks> */
 
diff --git a/src/lib-storage/index/imapc/imapc-storage.c b/src/lib-storage/index/imapc/imapc-storage.c
index ebe22b20aca..4c7e1a3eae2 100644
--- a/src/lib-storage/index/imapc/imapc-storage.c
+++ b/src/lib-storage/index/imapc/imapc-storage.c
@@ -55,6 +55,9 @@ static void imapc_untagged_status(const struct imapc_untagged_reply *reply,
 				  struct imapc_storage_client *client);
 static void imapc_untagged_namespace(const struct imapc_untagged_reply *reply,
 				     struct imapc_storage_client *client);
+static int imapc_mailbox_run_status(struct mailbox *box,
+				    enum mailbox_status_items items,
+				    struct mailbox_status *status_r);
 
 bool imap_resp_text_code_parse(const char *str, enum mail_error *error_r)
 {
@@ -72,6 +75,16 @@ bool imap_resp_text_code_parse(const char *str, enum mail_error *error_r)
 	return FALSE;
 }
 
+bool imapc_storage_has_modseqs(struct imapc_storage *storage)
+{
+	enum imapc_capability capa =
+		imapc_client_get_capabilities(storage->client->client);
+
+	return (capa & (IMAPC_CAPABILITY_CONDSTORE |
+			IMAPC_CAPABILITY_QRESYNC)) != 0 &&
+		IMAPC_HAS_FEATURE(storage, IMAPC_FEATURE_MODSEQ);
+}
+
 static struct mail_storage *imapc_storage_alloc(void)
 {
 	struct imapc_storage *storage;
@@ -603,6 +616,13 @@ static int imapc_mailbox_open(struct mailbox *box)
 		return -1;
 	}
 
+	if (imapc_storage_has_modseqs(mbox->storage)) {
+		if (!array_is_created(&mbox->rseq_modseqs))
+			i_array_init(&mbox->rseq_modseqs, 32);
+		else
+			array_clear(&mbox->rseq_modseqs);
+	}
+
 	if (imapc_mailbox_select(mbox) < 0) {
 		mailbox_close(box);
 		return -1;
@@ -635,6 +655,8 @@ static void imapc_mailbox_close(struct mailbox *box)
 		if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0)
 			mailbox_set_index_error(&mbox->box);
 	}
+	if (array_is_created(&mbox->rseq_modseqs))
+		array_free(&mbox->rseq_modseqs);
 	if (mbox->sync_view != NULL)
 		mail_index_view_close(&mbox->sync_view);
 	if (mbox->to_idle_delay != NULL)
@@ -727,6 +749,9 @@ static void imapc_untagged_status(const struct imapc_untagged_reply *reply,
 			status->uidvalidity = num;
 		else if (strcasecmp(key, "UNSEEN") == 0)
 			status->unseen = num;
+		else if (strcasecmp(key, "HIGHESTMODSEQ") == 0 &&
+			 imapc_storage_has_modseqs(storage))
+			status->highest_modseq = num;
 	}
 }
 
@@ -765,15 +790,32 @@ static void imapc_untagged_namespace(const struct imapc_untagged_reply *reply,
 	}
 }
 
-static void imapc_mailbox_get_selected_status(struct imapc_mailbox *mbox,
-					      enum mailbox_status_items items,
-					      struct mailbox_status *status_r)
+static int imapc_mailbox_get_selected_status(struct imapc_mailbox *mbox,
+					     enum mailbox_status_items items,
+					     struct mailbox_status *status_r)
 {
+	int ret = 0;
+
 	index_storage_get_open_status(&mbox->box, items, status_r);
 	if ((items & STATUS_PERMANENT_FLAGS) != 0)
 		status_r->permanent_flags = mbox->permanent_flags;
 	if ((items & STATUS_FIRST_RECENT_UID) != 0)
 		status_r->first_recent_uid = mbox->highest_nonrecent_uid + 1;
+	if ((items & STATUS_HIGHESTMODSEQ) != 0) {
+		/* FIXME: this doesn't work perfectly. we're now just returning
+		   the HIGHESTMODSEQ from the current index, which may or may
+		   not be correct. with QRESYNC enabled we could be returning
+		   sync_highestmodseq, but that would require implementing
+		   VANISHED replies. and without QRESYNC we'd have to issue
+		   STATUS (HIGHESTMODSEQ), which isn't efficient since we get
+		   here constantly (after every IMAP command). */
+	}
+	if (imapc_storage_has_modseqs(mbox->storage)) {
+		/* even if local indexes are only in memory, we still
+		   have modseqs on the IMAP server itself. */
+		status_r->nonpermanent_modseqs = FALSE;
+	}
+	return ret;
 }
 
 static int imapc_mailbox_delete(struct mailbox *box)
@@ -802,6 +844,9 @@ static int imapc_mailbox_run_status(struct mailbox *box,
 		str_append(str, " UIDVALIDITY");
 	if ((items & STATUS_UNSEEN) != 0)
 		str_append(str, " UNSEEN");
+	if ((items & STATUS_HIGHESTMODSEQ) != 0 &&
+	    imapc_storage_has_modseqs(mbox->storage))
+		str_append(str, " HIGHESTMODSEQ");
 
 	if (str_len(str) == 0) {
 		/* nothing requested */
@@ -833,14 +878,17 @@ static int imapc_mailbox_get_status(struct mailbox *box,
 		status_r->have_guids = TRUE;
 
 	if (box->opened) {
-		imapc_mailbox_get_selected_status(mbox, items, status_r);
+		if (imapc_mailbox_get_selected_status(mbox, items, status_r) < 0) {
+			/* can't do anything about this */
+		}
 	} else if ((items & (STATUS_FIRST_UNSEEN_SEQ | STATUS_KEYWORDS |
 			     STATUS_PERMANENT_FLAGS |
 			     STATUS_FIRST_RECENT_UID)) != 0) {
 		/* getting these requires opening the mailbox */
 		if (mailbox_open(box) < 0)
 			return -1;
-		imapc_mailbox_get_selected_status(mbox, items, status_r);
+		if (imapc_mailbox_get_selected_status(mbox, items, status_r) < 0)
+			return -1;
 	} else {
 		if (imapc_mailbox_run_status(box, items, status_r) < 0)
 			return -1;
diff --git a/src/lib-storage/index/imapc/imapc-storage.h b/src/lib-storage/index/imapc/imapc-storage.h
index 25e398f007a..f2f89aeaa05 100644
--- a/src/lib-storage/index/imapc/imapc-storage.h
+++ b/src/lib-storage/index/imapc/imapc-storage.h
@@ -108,9 +108,11 @@ struct imapc_mailbox {
 	enum mail_flags permanent_flags;
 	uint32_t highest_nonrecent_uid;
 
+	ARRAY(uint64_t) rseq_modseqs;
 	ARRAY_TYPE(uint32_t) delayed_expunged_uids;
 	uint32_t sync_uid_validity;
 	uint32_t sync_uid_next;
+	uint64_t sync_highestmodseq;
 	uint32_t sync_fetch_first_uid;
 	uint32_t sync_next_lseq;
 	uint32_t sync_next_rseq;
@@ -165,6 +167,7 @@ void imapc_mailbox_run_nofetch(struct imapc_mailbox *mbox);
 void imapc_mail_cache_free(struct imapc_mail_cache *cache);
 int imapc_mailbox_select(struct imapc_mailbox *mbox);
 
+bool imapc_storage_has_modseqs(struct imapc_storage *storage);
 bool imap_resp_text_code_parse(const char *str, enum mail_error *error_r);
 void imapc_copy_error_from_reply(struct imapc_storage *storage,
 				 enum mail_error default_error,
diff --git a/src/lib-storage/index/imapc/imapc-sync.c b/src/lib-storage/index/imapc/imapc-sync.c
index d14b12ccf51..fa3e66f6253 100644
--- a/src/lib-storage/index/imapc/imapc-sync.c
+++ b/src/lib-storage/index/imapc/imapc-sync.c
@@ -5,6 +5,7 @@
 #include "str.h"
 #include "imap-util.h"
 #include "mail-cache.h"
+#include "mail-index-modseq.h"
 #include "index-sync-private.h"
 #include "imapc-client.h"
 #include "imapc-msgmap.h"
@@ -245,6 +246,13 @@ static void imapc_sync_uid_next(struct imapc_sync_context *ctx)
 	}
 }
 
+static void imapc_sync_highestmodseq(struct imapc_sync_context *ctx)
+{
+	if (imapc_storage_has_modseqs(ctx->mbox->storage) &&
+	    mail_index_modseq_get_highest(ctx->sync_view) < ctx->mbox->sync_highestmodseq)
+		mail_index_update_highest_modseq(ctx->trans, ctx->mbox->sync_highestmodseq);
+}
+
 static void
 imapc_initial_sync_check(struct imapc_sync_context *ctx, bool nooped)
 {
@@ -311,6 +319,10 @@ imapc_sync_send_commands(struct imapc_sync_context *ctx, uint32_t first_uid)
 	string_t *cmd = t_str_new(64);
 
 	str_printfa(cmd, "UID FETCH %u:* (FLAGS", first_uid);
+	if (imapc_storage_has_modseqs(ctx->mbox->storage)) {
+		str_append(cmd, " MODSEQ");
+		mail_index_modseq_enable(ctx->mbox->box.index);
+	}
 	if (IMAPC_BOX_HAS_FEATURE(ctx->mbox, IMAPC_FEATURE_GMAIL_MIGRATION)) {
 		enum mailbox_info_flags flags;
 
@@ -393,8 +405,9 @@ static void imapc_sync_index(struct imapc_sync_context *ctx)
 		imapc_mailbox_run(mbox);
 	array_free(&ctx->expunged_uids);
 
-	/* add uidnext after all appends */
+	/* add uidnext & highestmodseq after all appends */
 	imapc_sync_uid_next(ctx);
+	imapc_sync_highestmodseq(ctx);
 
 	if (!ctx->failed)
 		imapc_sync_expunge_eom(ctx);

From 2335ab1f90e1e2ecdb856c0d18f5d823011c301e Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 22 Apr 2016 16:00:36 +0300
Subject: [PATCH 022/450] push-notification: Small optimization - don't lookup
 uidvalidity for each message.

---
 .../push-notification-txn-msg.c               | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/src/plugins/push-notification/push-notification-txn-msg.c b/src/plugins/push-notification/push-notification-txn-msg.c
index 65e64537ecd..ebc9e2fbf4f 100644
--- a/src/plugins/push-notification/push-notification-txn-msg.c
+++ b/src/plugins/push-notification/push-notification-txn-msg.c
@@ -45,7 +45,7 @@ push_notification_txn_msg_end(struct push_notification_txn *ptxn,
     struct push_notification_driver_txn **dtxn;
     struct seq_range_iter siter;
     struct mailbox_status status;
-    uint32_t uid;
+    uint32_t uid, uid_validity;
     struct push_notification_txn_msg *value;
 
     if (!hash_table_is_created(ptxn->messages)) {
@@ -55,20 +55,21 @@ push_notification_txn_msg_end(struct push_notification_txn *ptxn,
     hiter = hash_table_iterate_init(ptxn->messages);
     seq_range_array_iter_init(&siter, &changes->saved_uids);
 
+    /* uid_validity is only set in changes if message is new. */
+    if (changes->uid_validity == 0) {
+        mailbox_get_open_status(ptxn->mbox, STATUS_UIDVALIDITY, &status);
+        uid_validity = status.uidvalidity;
+    } else {
+        uid_validity = changes->uid_validity;
+    }
+
     while (hash_table_iterate(hiter, ptxn->messages, &key, &value)) {
         if (value->uid == 0) {
             if (seq_range_array_iter_nth(&siter, value->seq, &uid)) {
                 value->uid = uid;
             }
         }
-
-        /* uid_validity is only set in changes if message is new. */
-        if (changes->uid_validity == 0) {
-            mailbox_get_open_status(ptxn->mbox, STATUS_UIDVALIDITY, &status);
-            value->uid_validity = status.uidvalidity;
-        } else {
-            value->uid_validity = changes->uid_validity;
-        }
+        value->uid_validity = uid_validity;
 
         array_foreach_modifiable(&ptxn->drivers, dtxn) {
             if ((*dtxn)->duser->driver->v.process_msg != NULL) {

From 2f1be1ff67f5a30a6ae3e3536f570b52a6378828 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 19 Apr 2016 16:55:02 +0300
Subject: [PATCH 023/450] lib-http: Include information about number of request
 attempts and its timing in response reason.

Because the reason is usually logged as part of the error string, this
causes all of the error messages to include the attempts count and how long
the requests took in total. This should make it easier to understand problems
in error logs.

http_client_request_set_preserve_exact_reason() can be used to disable
modifying the reason string. This may also apply to other reason
modifications that may be done in the future.
---
 src/lib-http/http-client-private.h |  1 +
 src/lib-http/http-client-request.c | 18 +++++++++++++++++-
 src/lib-http/http-client.h         |  1 +
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/lib-http/http-client-private.h b/src/lib-http/http-client-private.h
index f34a4ceb1d8..27ad2d1fc4b 100644
--- a/src/lib-http/http-client-private.h
+++ b/src/lib-http/http-client-private.h
@@ -122,6 +122,7 @@ struct http_client_request {
 	unsigned int connect_tunnel:1;
 	unsigned int connect_direct:1;
 	unsigned int ssl_tunnel:1;
+	unsigned int preserve_exact_reason:1;
 };
 
 struct http_client_connection {
diff --git a/src/lib-http/http-client-request.c b/src/lib-http/http-client-request.c
index cde2db97c64..96120583587 100644
--- a/src/lib-http/http-client-request.c
+++ b/src/lib-http/http-client-request.c
@@ -246,6 +246,11 @@ void http_client_request_set_urgent(struct http_client_request *req)
 	req->urgent = TRUE;
 }
 
+void http_client_request_set_preserve_exact_reason(struct http_client_request *req)
+{
+	req->preserve_exact_reason = TRUE;
+}
+
 void http_client_request_add_header(struct http_client_request *req,
 				    const char *key, const char *value)
 {
@@ -1016,7 +1021,18 @@ bool http_client_request_callback(struct http_client_request *req,
 
 	req->callback = NULL;
 	if (callback != NULL) {
-		callback(response, req->context);
+		struct http_response response_copy = *response;
+
+		if (req->attempts > 0 && !req->preserve_exact_reason) {
+			unsigned int total_msecs =
+				timeval_diff_msecs(&ioloop_timeval, &req->submit_time);
+			response_copy.reason = t_strdup_printf(
+				"%s (%u attempts in %u.%03u secs)",
+				response_copy.reason, req->attempts,
+				total_msecs/1000, total_msecs%1000);
+		}
+
+		callback(&response_copy, req->context);
 		if (req->attempts != orig_attempts) {
 			/* retrying */
 			req->callback = callback;
diff --git a/src/lib-http/http-client.h b/src/lib-http/http-client.h
index ddc221e538a..fba5d1e0363 100644
--- a/src/lib-http/http-client.h
+++ b/src/lib-http/http-client.h
@@ -194,6 +194,7 @@ void http_client_request_set_port(struct http_client_request *req,
 void http_client_request_set_ssl(struct http_client_request *req,
 	bool ssl);
 void http_client_request_set_urgent(struct http_client_request *req);
+void http_client_request_set_preserve_exact_reason(struct http_client_request *req);
 
 void http_client_request_add_header(struct http_client_request *req,
 				    const char *key, const char *value);

From de73aa2bdf45647d3d8bb00f045840f9dafc6861 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 21 Apr 2016 21:45:02 +0300
Subject: [PATCH 024/450] lib-ssl-iostream: Return stacked errors as single
 combined string.

Instead of logging stacked errors as separate log lines, which don't provide
any context of what created them.
---
 .../iostream-openssl-common.c                 | 25 ++++++++++++++-----
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/src/lib-ssl-iostream/iostream-openssl-common.c b/src/lib-ssl-iostream/iostream-openssl-common.c
index 572689763d7..439ccfd08c7 100644
--- a/src/lib-ssl-iostream/iostream-openssl-common.c
+++ b/src/lib-ssl-iostream/iostream-openssl-common.c
@@ -1,6 +1,7 @@
 /* Copyright (c) 2009-2016 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
+#include "str.h"
 #include "iostream-openssl.h"
 
 #include <openssl/x509v3.h>
@@ -185,8 +186,9 @@ static const char *ssl_err2str(unsigned long err, const char *data, int flags)
 
 const char *openssl_iostream_error(void)
 {
+	string_t *errstr = NULL;
 	unsigned long err;
-	const char *data;
+	const char *data, *final_error;
 	int flags;
 
 	while ((err = ERR_get_error_line_data(NULL, NULL, &data, &flags)) != 0) {
@@ -194,15 +196,26 @@ const char *openssl_iostream_error(void)
 			i_fatal_status(FATAL_OUTOFMEM, "OpenSSL malloc() failed");
 		if (ERR_peek_error() == 0)
 			break;
-		i_error("SSL: Stacked error: %s",
-			ssl_err2str(err, data, flags));
+		if (errstr == NULL)
+			errstr = t_str_new(128);
+		else
+			str_append(errstr, ", ");
+		str_append(errstr, ssl_err2str(err, data, flags));
 	}
 	if (err == 0) {
 		if (errno != 0)
-			return strerror(errno);
-		return "Unknown error";
+			final_error = strerror(errno);
+		else
+			final_error = "Unknown error";
+	} else {
+		final_error = ssl_err2str(err, data, flags);
+	}
+	if (errstr == NULL)
+		return final_error;
+	else {
+		str_printfa(errstr, ", %s", final_error);
+		return str_c(errstr);
 	}
-	return ssl_err2str(err, data, flags);
 }
 
 const char *openssl_iostream_key_load_error(void)

From 06c326243f9d65fd18b6fa1ca99d46c7f7113f98 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 20 Apr 2016 23:01:45 +0300
Subject: [PATCH 025/450] lib: var_expand() now expands %{nonexistent} to
 UNSUPPORTED_VARIABLE_nonexistent

Earlier it was expanded to "nonexistent}", which looked more like a bug.
This change hopefully makes it clear enough to understand when a variable isn't
supported.
---
 src/lib/test-var-expand.c | 5 ++++-
 src/lib/var-expand.c      | 6 ++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/lib/test-var-expand.c b/src/lib/test-var-expand.c
index 370199cb26c..d3f248a9ed4 100644
--- a/src/lib/test-var-expand.c
+++ b/src/lib/test-var-expand.c
@@ -52,7 +52,10 @@ static void test_var_expand_builtin(void)
 		{ "%50Hv", "1f" },
 		{ "%50Hw", "2e" },
 		{ "%50Nv", "25" },
-		{ "%50Nw", "e" }
+		{ "%50Nw", "e" },
+
+		{ "%{nonexistent}", "UNSUPPORTED_VARIABLE_nonexistent" },
+		{ "%{nonexistent:default}", "UNSUPPORTED_VARIABLE_nonexistent" },
 	};
 	static struct var_expand_table table[] = {
 		{ 'v', "value", NULL },
diff --git a/src/lib/var-expand.c b/src/lib/var-expand.c
index fee44c9c4bc..5d06cebee05 100644
--- a/src/lib/var-expand.c
+++ b/src/lib/var-expand.c
@@ -232,6 +232,8 @@ var_expand_long(const struct var_expand_table *table,
 			data = "";
 		value = var_expand_func(func_table, key, data, context);
 	}
+	if (value == NULL)
+		return t_strdup_printf("UNSUPPORTED_VARIABLE_%s", key);
 	return value;
 }
 
@@ -325,8 +327,8 @@ void var_expand_with_funcs(string_t *dest, const char *str,
 				len = end - (str + 1);
 				var = var_expand_long(table, func_table,
 						      str+1, len, context);
-				if (var != NULL)
-					str = end;
+				i_assert(var != NULL);
+				str = end;
 			} else if (table != NULL) {
 				for (t = table; !TABLE_LAST(t); t++) {
 					if (t->key == *str) {

From bdff78333837f45abc804ca49ad1156e8ee4066c Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 21 Apr 2016 17:52:44 +0300
Subject: [PATCH 026/450] lib-fs: Added fs-test backend for helping with
 creation of fs-wrapper unit tests.

---
 src/lib-fs/Makefile.am      |   3 +
 src/lib-fs/fs-api-private.h |   1 +
 src/lib-fs/fs-api.c         |   1 +
 src/lib-fs/fs-test.c        | 355 ++++++++++++++++++++++++++++++++++++
 src/lib-fs/fs-test.h        |  38 ++++
 5 files changed, 398 insertions(+)
 create mode 100644 src/lib-fs/fs-test.c
 create mode 100644 src/lib-fs/fs-test.h

diff --git a/src/lib-fs/Makefile.am b/src/lib-fs/Makefile.am
index aed58c18f37..5df3c5dd458 100644
--- a/src/lib-fs/Makefile.am
+++ b/src/lib-fs/Makefile.am
@@ -2,6 +2,7 @@ noinst_LTLIBRARIES = libfs.la
 
 AM_CPPFLAGS = \
 	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-test \
 	-I$(top_srcdir)/src/lib-dict \
 	-I$(top_srcdir)/src/lib-ssl-iostream \
 	-DMODULE_DIR=\""$(moduledir)"\"
@@ -12,6 +13,7 @@ libfs_la_SOURCES = \
 	fs-metawrap.c \
 	fs-randomfail.c \
 	fs-posix.c \
+	fs-test.c \
 	fs-sis.c \
 	fs-sis-common.c \
 	fs-sis-queue.c \
@@ -25,6 +27,7 @@ headers = \
 	fs-api.h \
 	fs-api-private.h \
 	fs-sis-common.h \
+	fs-test.h \
 	istream-fs-file.h \
 	istream-fs-stats.h \
 	istream-metawrap.h \
diff --git a/src/lib-fs/fs-api-private.h b/src/lib-fs/fs-api-private.h
index 8e6d173f5eb..6f931c19c29 100644
--- a/src/lib-fs/fs-api-private.h
+++ b/src/lib-fs/fs-api-private.h
@@ -145,6 +145,7 @@ extern const struct fs fs_class_randomfail;
 extern const struct fs fs_class_metawrap;
 extern const struct fs fs_class_sis;
 extern const struct fs fs_class_sis_queue;
+extern const struct fs fs_class_test;
 
 void fs_class_register(const struct fs *fs_class);
 
diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c
index 921485d47ae..b50fbe0f53c 100644
--- a/src/lib-fs/fs-api.c
+++ b/src/lib-fs/fs-api.c
@@ -74,6 +74,7 @@ static void fs_classes_init(void)
 	fs_class_register(&fs_class_metawrap);
 	fs_class_register(&fs_class_sis);
 	fs_class_register(&fs_class_sis_queue);
+	fs_class_register(&fs_class_test);
 	lib_atexit(fs_classes_deinit);
 }
 
diff --git a/src/lib-fs/fs-test.c b/src/lib-fs/fs-test.c
new file mode 100644
index 00000000000..211b61f4129
--- /dev/null
+++ b/src/lib-fs/fs-test.c
@@ -0,0 +1,355 @@
+/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "ostream.h"
+#include "test-common.h"
+#include "fs-test.h"
+
+static struct fs *fs_test_alloc(void)
+{
+	struct test_fs *fs;
+
+	fs = i_new(struct test_fs, 1);
+	fs->fs = fs_class_test;
+	i_array_init(&fs->iter_files, 32);
+	return &fs->fs;
+}
+
+static int
+fs_test_init(struct fs *_fs ATTR_UNUSED, const char *args ATTR_UNUSED,
+	     const struct fs_settings *set ATTR_UNUSED)
+{
+	return 0;
+}
+
+static void fs_test_deinit(struct fs *_fs)
+{
+	struct test_fs *fs = (struct test_fs *)_fs;
+
+	array_free(&fs->iter_files);
+	i_free(fs);
+}
+
+static enum fs_properties fs_test_get_properties(struct fs *_fs)
+{
+	struct test_fs *fs = (struct test_fs *)_fs;
+
+	return fs->properties;
+}
+
+static struct fs_file *
+fs_test_file_init(struct fs *_fs, const char *path,
+		  enum fs_open_mode mode, enum fs_open_flags flags ATTR_UNUSED)
+{
+	struct test_fs_file *file;
+
+	file = i_new(struct test_fs_file, 1);
+	file->file.fs = _fs;
+	file->file.path = i_strdup(path);
+	file->mode = mode;
+	file->contents = buffer_create_dynamic(default_pool, 1024);
+	file->exists = TRUE;
+	file->seekable = TRUE;
+	return &file->file;
+}
+
+static void fs_test_file_deinit(struct fs_file *_file)
+{
+	struct test_fs_file *file = (struct test_fs_file *)_file;
+
+	buffer_free(&file->contents);
+	i_free(file->file.path);
+	i_free(file);
+}
+
+static void fs_test_file_close(struct fs_file *_file)
+{
+	struct test_fs_file *file = (struct test_fs_file *)_file;
+
+	file->closed = TRUE;
+}
+
+static const char *fs_test_file_get_path(struct fs_file *_file)
+{
+	return _file->path;
+}
+
+static void
+fs_test_set_async_callback(struct fs_file *_file,
+			   fs_file_async_callback_t *callback,
+			   void *context)
+{
+	struct test_fs_file *file = (struct test_fs_file *)_file;
+
+	file->async_callback = callback;
+	file->async_context = context;
+}
+
+static int fs_test_wait_async(struct fs *_fs ATTR_UNUSED)
+{
+	return 0;
+}
+
+static void
+fs_test_set_metadata(struct fs_file *_file, const char *key,
+		     const char *value)
+{
+	fs_default_set_metadata(_file, key, value);
+}
+
+static int
+fs_test_get_metadata(struct fs_file *_file,
+		     const ARRAY_TYPE(fs_metadata) **metadata_r)
+{
+	fs_metadata_init(_file);
+	*metadata_r = &_file->metadata;
+	return 0;
+}
+
+static bool fs_test_prefetch(struct fs_file *_file ATTR_UNUSED,
+			     uoff_t length ATTR_UNUSED)
+{
+	struct test_fs_file *file = (struct test_fs_file *)_file;
+
+	file->prefetched = TRUE;
+	return TRUE;
+}
+
+static void fs_test_stream_destroyed(struct test_fs_file *file)
+{
+	i_assert(file->input != NULL);
+	file->input = NULL;
+}
+
+static struct istream *
+fs_test_read_stream(struct fs_file *_file, size_t max_buffer_size ATTR_UNUSED)
+{
+	struct test_fs_file *file = (struct test_fs_file *)_file;
+	struct istream *input;
+
+	i_assert(file->input == NULL);
+
+	if (!file->exists)
+		return i_stream_create_error(ENOENT);
+	input = test_istream_create_data(file->contents->data,
+					 file->contents->used);
+	i_stream_add_destroy_callback(input, fs_test_stream_destroyed, file);
+	if (!file->seekable)
+		input->seekable = FALSE;
+	file->input = input;
+	return input;
+}
+
+static void fs_test_write_stream(struct fs_file *_file)
+{
+	struct test_fs_file *file = (struct test_fs_file *)_file;
+
+	i_assert(_file->output == NULL);
+
+	buffer_set_used_size(file->contents, 0);
+	_file->output = o_stream_create_buffer(file->contents);
+}
+
+static int fs_test_write_stream_finish(struct fs_file *_file, bool success)
+{
+	struct test_fs_file *file = (struct test_fs_file *)_file;
+
+	if (!success)
+		buffer_set_used_size(file->contents, 0);
+	return success ? 1 : -1;
+}
+
+static int
+fs_test_lock(struct fs_file *_file, unsigned int secs ATTR_UNUSED,
+	     struct fs_lock **lock_r)
+{
+	struct test_fs_file *file = (struct test_fs_file *)_file;
+
+	if (file->locked)
+		return 0;
+	file->locked = TRUE;
+	*lock_r = i_new(struct fs_lock, 1);
+	(*lock_r)->file = _file;
+	return 1;
+}
+
+static void fs_test_unlock(struct fs_lock *lock)
+{
+	struct test_fs_file *file = (struct test_fs_file *)lock->file;
+
+	file->locked = FALSE;
+	i_free(lock);
+}
+
+static int fs_test_exists(struct fs_file *_file)
+{
+	struct test_fs_file *file = (struct test_fs_file *)_file;
+
+	return file->exists ? 1 : 0;
+}
+
+static int fs_test_stat(struct fs_file *_file, struct stat *st_r)
+{
+	struct test_fs_file *file = (struct test_fs_file *)_file;
+
+	if (!file->exists) {
+		errno = ENOENT;
+		return -1;
+	}
+	memset(st_r, 0, sizeof(*st_r));
+	st_r->st_size = file->contents->used;
+	return 0;
+}
+
+static int fs_test_copy(struct fs_file *_src, struct fs_file *_dest)
+{
+	struct test_fs_file *src = (struct test_fs_file *)_src;
+	struct test_fs_file *dest = (struct test_fs_file *)_dest;
+
+	if (!src->exists) {
+		errno = ENOENT;
+		return -1;
+	}
+	buffer_set_used_size(dest->contents, 0);
+	buffer_append_buf(dest->contents, src->contents, 0, (size_t)-1);
+	dest->exists = TRUE;
+	return 0;
+}
+
+static int fs_test_rename(struct fs_file *_src, struct fs_file *_dest)
+{
+	struct test_fs_file *src = (struct test_fs_file *)_src;
+
+	if (fs_test_copy(_src, _dest) < 0)
+		return -1;
+	src->exists = FALSE;
+	return 0;
+}
+
+static int fs_test_delete(struct fs_file *_file)
+{
+	struct test_fs_file *file = (struct test_fs_file *)_file;
+
+	if (!file->exists) {
+		errno = ENOENT;
+		return -1;
+	}
+	return 0;
+}
+
+static struct fs_iter *
+fs_test_iter_init(struct fs *_fs, const char *path,
+		  enum fs_iter_flags flags)
+{
+	struct test_fs *fs = (struct test_fs *)_fs;
+	struct test_fs_iter *iter;
+
+	iter = i_new(struct test_fs_iter, 1);
+	iter->iter.fs = _fs;
+	iter->iter.flags = flags;
+	iter->prefix = i_strdup(path);
+	iter->prefix_len = strlen(iter->prefix);
+	iter->prev_dir = i_strdup("");
+	array_sort(&fs->iter_files, i_strcmp_p);
+	return &iter->iter;
+}
+
+static const char *fs_test_iter_next(struct fs_iter *_iter)
+{
+	struct test_fs_iter *iter = (struct test_fs_iter *)_iter;
+	struct test_fs *fs = (struct test_fs *)_iter->fs;
+	const char *const *files, *p;
+	unsigned int count, len, prev_dir_len = strlen(iter->prev_dir);
+
+	files = array_get(&fs->iter_files, &count);
+	for (; iter->idx < count; iter->idx++) {
+		const char *fname = files[iter->idx];
+
+		if (strncmp(fname, iter->prefix, iter->prefix_len) != 0)
+			continue;
+		p = strrchr(fname, '/');
+		if ((_iter->flags & FS_ITER_FLAG_DIRS) == 0) {
+			if (p == NULL)
+				return fname;
+			if (p[1] == '\0')
+				continue; /* dir/ */
+			return p+1;
+		}
+
+		if (p == NULL)
+			continue;
+		len = p - fname;
+		if (len == 0)
+			continue;
+		if (len == prev_dir_len &&
+		    strncmp(fname, iter->prev_dir, len) == 0)
+			continue;
+		i_free(iter->prev_dir);
+		iter->prev_dir = i_strndup(fname, len);
+		return iter->prev_dir;
+	}
+	return NULL;
+}
+
+static int fs_test_iter_deinit(struct fs_iter *_iter)
+{
+	struct test_fs_iter *iter = (struct test_fs_iter *)_iter;
+	int ret = iter->failed ? -1 : 0;
+
+	i_free(iter->prefix);
+	i_free(iter);
+	return ret;
+}
+
+struct test_fs_file *test_fs_file_get(struct fs *fs, unsigned int n)
+{
+	struct fs_file *file;
+
+	while (strcmp(fs->name, "test") != 0) {
+		i_assert(fs->parent != NULL);
+		fs = fs->parent;
+	}
+
+	file = fs->files;
+	for (; n > 0; n--) {
+		i_assert(file != NULL);
+		file = file->next;
+	}
+	i_assert(file != NULL);
+	return (struct test_fs_file *)file;
+}
+
+const struct fs fs_class_test = {
+	.name = "test",
+	.v = {
+		fs_test_alloc,
+		fs_test_init,
+		fs_test_deinit,
+		fs_test_get_properties,
+		fs_test_file_init,
+		fs_test_file_deinit,
+		fs_test_file_close,
+		fs_test_file_get_path,
+		fs_test_set_async_callback,
+		fs_test_wait_async,
+		fs_test_set_metadata,
+		fs_test_get_metadata,
+		fs_test_prefetch,
+		NULL,
+		fs_test_read_stream,
+		NULL,
+		fs_test_write_stream,
+		fs_test_write_stream_finish,
+		fs_test_lock,
+		fs_test_unlock,
+		fs_test_exists,
+		fs_test_stat,
+		fs_test_copy,
+		fs_test_rename,
+		fs_test_delete,
+		fs_test_iter_init,
+		fs_test_iter_next,
+		fs_test_iter_deinit
+	}
+};
diff --git a/src/lib-fs/fs-test.h b/src/lib-fs/fs-test.h
new file mode 100644
index 00000000000..c5f94cc4f2c
--- /dev/null
+++ b/src/lib-fs/fs-test.h
@@ -0,0 +1,38 @@
+#ifndef FS_TEST_H
+#define FS_TEST_H
+
+#include "fs-api-private.h"
+
+struct test_fs {
+	struct fs fs;
+	enum fs_properties properties;
+	ARRAY_TYPE(const_string) iter_files;
+};
+
+struct test_fs_file {
+	struct fs_file file;
+	enum fs_open_mode mode;
+
+	fs_file_async_callback_t *async_callback;
+	void *async_context;
+
+	buffer_t *contents;
+	struct istream *input;
+
+	bool prefetched;
+	bool locked;
+	bool exists;
+	bool seekable;
+	bool closed;
+};
+
+struct test_fs_iter {
+	struct fs_iter iter;
+	char *prefix, *prev_dir;
+	unsigned int prefix_len, idx;
+	bool failed;
+};
+
+struct test_fs_file *test_fs_file_get(struct fs *fs, unsigned int n);
+
+#endif

From 5c201395ff5bf096b341581d31b0e507aa47b2cc Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 21 Apr 2016 17:54:54 +0300
Subject: [PATCH 027/450] lib-fs: Added initial fs-metawrap unit test.

---
 src/lib-fs/Makefile.am        | 25 ++++++++++++++++
 src/lib-fs/test-fs-metawrap.c | 56 +++++++++++++++++++++++++++++++++++
 2 files changed, 81 insertions(+)
 create mode 100644 src/lib-fs/test-fs-metawrap.c

diff --git a/src/lib-fs/Makefile.am b/src/lib-fs/Makefile.am
index 5df3c5dd458..9a445cb949f 100644
--- a/src/lib-fs/Makefile.am
+++ b/src/lib-fs/Makefile.am
@@ -36,3 +36,28 @@ headers = \
 
 pkginc_libdir=$(pkgincludedir)
 pkginc_lib_HEADERS = $(headers)
+
+noinst_PROGRAMS = $(test_programs)
+
+test_programs = \
+	test-fs-metawrap
+
+test_deps = \
+	$(noinst_LTLIBRARIES) \
+	../lib-dict/libdict.la \
+	../lib-test/libtest.la \
+	../lib/liblib.la
+
+test_libs = \
+	$(test_deps) \
+	$(MODULE_LIBS)
+
+test_fs_metawrap_SOURCES = test-fs-metawrap.c
+test_fs_metawrap_LDADD = $(test_libs)
+test_fs_metawrap_DEPENDENCIES = $(test_deps)
+
+check: check-am check-test
+check-test: all-am
+	for bin in $(test_programs); do \
+	  if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
+	done
diff --git a/src/lib-fs/test-fs-metawrap.c b/src/lib-fs/test-fs-metawrap.c
new file mode 100644
index 00000000000..b09c92dad11
--- /dev/null
+++ b/src/lib-fs/test-fs-metawrap.c
@@ -0,0 +1,56 @@
+/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "istream.h"
+#include "fs-test.h"
+#include "test-common.h"
+
+static void test_fs_metawrap_stat(void)
+{
+	struct fs_settings fs_set;
+	struct fs *fs;
+	struct fs_file *file;
+	struct test_fs_file *test_file;
+	struct istream *input;
+	struct stat st;
+	const char *error;
+	unsigned int i;
+
+	test_begin("fs metawrap stat");
+
+	memset(&fs_set, 0, sizeof(fs_set));
+	if (fs_init("metawrap", "test", &fs_set, &fs, &error) < 0)
+		i_fatal("fs_init() failed: %s", error);
+
+	for (i = 0; i < 2; i++) {
+		file = fs_file_init(fs, "foo", FS_OPEN_MODE_READONLY);
+
+		test_file = test_fs_file_get(fs, 0);
+		str_append(test_file->contents, "key:value\n\n12345678901234567890");
+
+		if (i == 0) {
+			input = fs_read_stream(file, 2);
+			test_istream_set_max_buffer_size(test_file->input, 2);
+		} else {
+			input = NULL;
+		}
+
+		test_assert_idx(fs_stat(file, &st) == 0 && st.st_size == 20, i);
+
+		if (input != NULL)
+			i_stream_unref(&input);
+		fs_file_deinit(&file);
+	}
+	fs_deinit(&fs);
+	test_end();
+}
+
+int main(void)
+{
+	static void (*test_functions[])(void) = {
+		test_fs_metawrap_stat,
+		NULL
+	};
+	return test_run(test_functions);
+}

From 973c6d6d86b84541c195c01ae32778444884d401 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 21 Apr 2016 17:55:51 +0300
Subject: [PATCH 028/450] lib-fs: fs-metawrap stat() and get_metadata(): use
 existing istream if possible

This may reduce reads from parent fs, but it's at least required for the
following commit.
---
 src/lib-fs/fs-metawrap.c | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/src/lib-fs/fs-metawrap.c b/src/lib-fs/fs-metawrap.c
index a8c9f8ace52..ee187b96098 100644
--- a/src/lib-fs/fs-metawrap.c
+++ b/src/lib-fs/fs-metawrap.c
@@ -199,14 +199,30 @@ fs_metawrap_get_metadata(struct fs_file *_file,
 			 const ARRAY_TYPE(fs_metadata) **metadata_r)
 {
 	struct metawrap_fs_file *file = (struct metawrap_fs_file *)_file;
+	ssize_t ret;
 	char c;
 
 	if (!file->fs->wrap_metadata)
 		return fs_get_metadata(file->super, metadata_r);
 
-	if (!file->metadata_read) {
+	if (file->metadata_read) {
+		/* we have the metadata */
+	} else if (file->input == NULL) {
 		if (fs_read(_file, &c, 1) < 0)
 			return -1;
+	} else {
+		/* use the existing istream to read it */
+		while ((ret = i_stream_read(file->input)) == 0) {
+			if (file->metadata_read)
+				break;
+
+			i_assert(!file->input->blocking);
+			if (fs_wait_async(_file->fs) < 0)
+				return -1;
+		}
+		if (ret == -1)
+			return -1;
+		i_assert(file->metadata_read);
 	}
 	*metadata_r = &_file->metadata;
 	return 0;
@@ -469,7 +485,12 @@ static int fs_metawrap_stat(struct fs_file *_file, struct stat *st_r)
 		return 0;
 	}
 
-	input = fs_read_stream(_file, IO_BLOCK_SIZE);
+	if (file->input == NULL)
+		input = fs_read_stream(_file, IO_BLOCK_SIZE);
+	else {
+		input = file->input;
+		i_stream_ref(input);
+	}
 	if ((ret = i_stream_get_size(input, TRUE, &input_size)) < 0) {
 		fs_set_error(_file->fs, "i_stream_get_size(%s) failed: %s",
 			     fs_file_path(_file), i_stream_get_error(input));

From 412fb67b918c4511f54a621fcb20bb3130e95682 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 22 Apr 2016 16:25:00 +0300
Subject: [PATCH 029/450] dsync: Fixed crash when rename algorithm thinks it
 has gone to infinite loop.

ctx->brain was NULL, so trying to access it crashed.
---
 src/doveadm/dsync/dsync-brain-mailbox-tree.c |  3 ++-
 src/doveadm/dsync/dsync-mailbox-tree-sync.c  | 19 +++++++++++++------
 src/doveadm/dsync/dsync-mailbox-tree.h       |  2 +-
 3 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/src/doveadm/dsync/dsync-brain-mailbox-tree.c b/src/doveadm/dsync/dsync-brain-mailbox-tree.c
index 674cb0a4868..786efa50531 100644
--- a/src/doveadm/dsync/dsync-brain-mailbox-tree.c
+++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c
@@ -310,7 +310,8 @@ static void dsync_brain_mailbox_trees_sync(struct dsync_brain *brain)
 			break;
 		}
 	}
-	dsync_mailbox_trees_sync_deinit(&ctx);
+	if (dsync_mailbox_trees_sync_deinit(&ctx) < 0)
+		brain->failed = TRUE;
 }
 
 bool dsync_brain_recv_mailbox_tree(struct dsync_brain *brain)
diff --git a/src/doveadm/dsync/dsync-mailbox-tree-sync.c b/src/doveadm/dsync/dsync-mailbox-tree-sync.c
index 64fa3f7adb0..3ad3eec04e1 100644
--- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c
+++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c
@@ -27,13 +27,13 @@ struct dsync_mailbox_tree_bfs_iter {
 
 struct dsync_mailbox_tree_sync_ctx {
 	pool_t pool;
-	struct dsync_brain *brain;
 	struct dsync_mailbox_tree *local_tree, *remote_tree;
 	enum dsync_mailbox_trees_sync_type sync_type;
 	enum dsync_mailbox_trees_sync_flags sync_flags;
 
 	ARRAY(struct dsync_mailbox_tree_sync_change) changes;
 	unsigned int change_idx;
+	bool failed;
 };
 
 static struct dsync_mailbox_tree_bfs_iter *
@@ -1075,7 +1075,7 @@ sync_rename_temp_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
 	return FALSE;
 }
 
-static void
+static int
 dsync_mailbox_tree_handle_renames(struct dsync_mailbox_tree_sync_ctx *ctx)
 {
 	unsigned int count = 0;
@@ -1095,11 +1095,12 @@ dsync_mailbox_tree_handle_renames(struct dsync_mailbox_tree_sync_ctx *ctx)
 
 	if (changed) {
 		i_error("BUG: Mailbox renaming algorithm got into a potentially infinite loop, aborting");
-		ctx->brain->failed = TRUE;
+		return -1;
 	}
 
 	while (sync_rename_temp_mailboxes(ctx, ctx->local_tree, &ctx->local_tree->root)) ;
 	while (sync_rename_temp_mailboxes(ctx, ctx->remote_tree, &ctx->remote_tree->root)) ;
+	return 0;
 }
 
 static bool sync_is_wrong_mailbox(struct dsync_mailbox_node *node,
@@ -1411,8 +1412,12 @@ dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree,
 
 	dsync_mailbox_tree_update_child_timestamps(&local_tree->root, 0);
 	dsync_mailbox_tree_update_child_timestamps(&remote_tree->root, 0);
-	if ((sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES) == 0)
-		dsync_mailbox_tree_handle_renames(ctx);
+	if ((sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES) == 0) {
+		if (dsync_mailbox_tree_handle_renames(ctx) < 0) {
+			ctx->failed = TRUE;
+			return ctx;
+		}
+	}
 
 	/* if we're not doing a two-way sync, delete now any mailboxes, which
 	   a) shouldn't exist, b) doesn't have a matching GUID/UIDVALIDITY,
@@ -1438,12 +1443,14 @@ dsync_mailbox_trees_sync_next(struct dsync_mailbox_tree_sync_ctx *ctx)
 	return array_idx(&ctx->changes, ctx->change_idx++);
 }
 
-void dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **_ctx)
+int dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **_ctx)
 {
 	struct dsync_mailbox_tree_sync_ctx *ctx = *_ctx;
+	int ret = ctx->failed ? -1 : 0;
 
 	*_ctx = NULL;
 
 	array_free(&ctx->changes);
 	pool_unref(&ctx->pool);
+	return ret;
 }
diff --git a/src/doveadm/dsync/dsync-mailbox-tree.h b/src/doveadm/dsync/dsync-mailbox-tree.h
index af6ce369b99..d317aaa34f0 100644
--- a/src/doveadm/dsync/dsync-mailbox-tree.h
+++ b/src/doveadm/dsync/dsync-mailbox-tree.h
@@ -194,7 +194,7 @@ dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree,
 			      enum dsync_mailbox_trees_sync_flags sync_flags);
 const struct dsync_mailbox_tree_sync_change *
 dsync_mailbox_trees_sync_next(struct dsync_mailbox_tree_sync_ctx *ctx);
-void dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **ctx);
+int dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **ctx);
 
 const char *dsync_mailbox_node_to_string(const struct dsync_mailbox_node *node);
 const char *

From 66da5866eb19271378d62facf4a1fea0059e9e72 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 22 Apr 2016 16:28:56 +0300
Subject: [PATCH 030/450] dsync: Determine better when rename-algorithm might
 have gotten stuck.

A hardcoded value of 100 isn't necessarily enough if there are a lot of
mailboxes with a lot of renames. Base the max count on the total number
of mailboxes on both local and remote. And just in case multiple it by 3.
Probably smaller number would be fine too.
---
 src/doveadm/dsync/dsync-mailbox-tree-sync.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/doveadm/dsync/dsync-mailbox-tree-sync.c b/src/doveadm/dsync/dsync-mailbox-tree-sync.c
index 3ad3eec04e1..aee5f6425b7 100644
--- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c
+++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c
@@ -15,8 +15,6 @@
 #define TEMP_SUFFIX_MAX_LEN (sizeof("temp-")-1 + 8)
 #define TEMP_SUFFIX_FORMAT "temp-%x"
 
-#define MAX_RENAMES 100
-
 struct dsync_mailbox_tree_bfs_iter {
 	struct dsync_mailbox_tree *tree;
 
@@ -30,6 +28,7 @@ struct dsync_mailbox_tree_sync_ctx {
 	struct dsync_mailbox_tree *local_tree, *remote_tree;
 	enum dsync_mailbox_trees_sync_type sync_type;
 	enum dsync_mailbox_trees_sync_flags sync_flags;
+	unsigned int combined_mailboxes_count;
 
 	ARRAY(struct dsync_mailbox_tree_sync_change) changes;
 	unsigned int change_idx;
@@ -237,6 +236,7 @@ sync_tree_sort_and_delete_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
 				sync_set_node_deleted(tree, node);
 			}
 		}
+		ctx->combined_mailboxes_count++;
 		array_append(&siblings, &node, 1);
 	}
 	sort_siblings(&siblings);
@@ -1078,9 +1078,10 @@ sync_rename_temp_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
 static int
 dsync_mailbox_tree_handle_renames(struct dsync_mailbox_tree_sync_ctx *ctx)
 {
-	unsigned int count = 0;
+	unsigned int max_renames, count = 0;
 	bool changed;
 
+	max_renames = ctx->combined_mailboxes_count * 3;
 	do {
 		T_BEGIN {
 			changed = sync_rename_mailboxes(ctx, &ctx->local_tree->root,
@@ -1091,7 +1092,7 @@ dsync_mailbox_tree_handle_renames(struct dsync_mailbox_tree_sync_ctx *ctx)
 			i_debug("brain %c: -- Mailbox renamed, restart sync --",
 				(ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S');
 		}
-	} while (changed && ++count <= MAX_RENAMES);
+	} while (changed && ++count <= max_renames);
 
 	if (changed) {
 		i_error("BUG: Mailbox renaming algorithm got into a potentially infinite loop, aborting");

From ff1f5155e8707fdff18f4311ce6c901a1de4e769 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 22 Apr 2016 16:39:49 +0300
Subject: [PATCH 031/450] dsync: Fixed assert-crash in rename algorithm

Fixes a crash:
Panic: file dsync-mailbox-tree-sync.c: line 1308 (sync_mailbox_child_dirs): assertion failed: (!node_is_existent(local_node))
---
 src/doveadm/dsync/dsync-mailbox-tree-sync.c   | 29 +++++++++++++++----
 .../dsync/test-dsync-mailbox-tree-sync.c      | 23 +++++++++++++++
 2 files changed, 46 insertions(+), 6 deletions(-)

diff --git a/src/doveadm/dsync/dsync-mailbox-tree-sync.c b/src/doveadm/dsync/dsync-mailbox-tree-sync.c
index aee5f6425b7..3b0937927a4 100644
--- a/src/doveadm/dsync/dsync-mailbox-tree-sync.c
+++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c
@@ -1039,12 +1039,12 @@ sync_rename_delete_node_dirs(struct dsync_mailbox_tree_sync_ctx *ctx,
 static bool
 sync_rename_temp_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
 			   struct dsync_mailbox_tree *tree,
-			   struct dsync_mailbox_node *node)
+			   struct dsync_mailbox_node *node, bool *renames_r)
 {
 	const char *reason;
 
 	for (; node != NULL; node = node->next) {
-		while (sync_rename_temp_mailboxes(ctx, tree, node->first_child)) ;
+		while (sync_rename_temp_mailboxes(ctx, tree, node->first_child, renames_r)) ;
 
 		if (!node->sync_temporary_name) {
 		} else if (dsync_mailbox_node_is_dir(node) &&
@@ -1061,6 +1061,7 @@ sync_rename_temp_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
 			sync_rename_delete_node_dirs(ctx, tree, node);
 		} else {
 			T_BEGIN {
+				*renames_r = TRUE;
 				sync_rename_temp_mailbox_node(tree, node, &reason);
 				if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0) {
 					i_debug("brain %c: %s mailbox %s: %s",
@@ -1076,11 +1077,14 @@ sync_rename_temp_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
 }
 
 static int
-dsync_mailbox_tree_handle_renames(struct dsync_mailbox_tree_sync_ctx *ctx)
+dsync_mailbox_tree_handle_renames(struct dsync_mailbox_tree_sync_ctx *ctx,
+				  bool *renames_r)
 {
 	unsigned int max_renames, count = 0;
 	bool changed;
 
+	*renames_r = FALSE;
+
 	max_renames = ctx->combined_mailboxes_count * 3;
 	do {
 		T_BEGIN {
@@ -1099,8 +1103,8 @@ dsync_mailbox_tree_handle_renames(struct dsync_mailbox_tree_sync_ctx *ctx)
 		return -1;
 	}
 
-	while (sync_rename_temp_mailboxes(ctx, ctx->local_tree, &ctx->local_tree->root)) ;
-	while (sync_rename_temp_mailboxes(ctx, ctx->remote_tree, &ctx->remote_tree->root)) ;
+	while (sync_rename_temp_mailboxes(ctx, ctx->local_tree, &ctx->local_tree->root, renames_r)) ;
+	while (sync_rename_temp_mailboxes(ctx, ctx->remote_tree, &ctx->remote_tree->root, renames_r)) ;
 	return 0;
 }
 
@@ -1391,6 +1395,8 @@ dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree,
 			      enum dsync_mailbox_trees_sync_flags sync_flags)
 {
 	struct dsync_mailbox_tree_sync_ctx *ctx;
+	unsigned int rename_counter = 0;
+	bool renames;
 	pool_t pool;
 
 	i_assert(hash_table_is_created(local_tree->guid_hash));
@@ -1406,6 +1412,9 @@ dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree,
 	ctx->sync_flags = sync_flags;
 	i_array_init(&ctx->changes, 128);
 
+again:
+	renames = FALSE;
+	ctx->combined_mailboxes_count = 0;
 	sync_tree_sort_and_delete_mailboxes(ctx, remote_tree,
 		sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY);
 	sync_tree_sort_and_delete_mailboxes(ctx, local_tree,
@@ -1414,7 +1423,7 @@ dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree,
 	dsync_mailbox_tree_update_child_timestamps(&local_tree->root, 0);
 	dsync_mailbox_tree_update_child_timestamps(&remote_tree->root, 0);
 	if ((sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES) == 0) {
-		if (dsync_mailbox_tree_handle_renames(ctx) < 0) {
+		if (dsync_mailbox_tree_handle_renames(ctx, &renames) < 0) {
 			ctx->failed = TRUE;
 			return ctx;
 		}
@@ -1432,6 +1441,14 @@ dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree,
 		sync_create_mailboxes(ctx, remote_tree);
 	if (sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE)
 		sync_create_mailboxes(ctx, local_tree);
+	if (renames && rename_counter++ <= ctx->combined_mailboxes_count*3) {
+		/* this rename algorithm is just horrible. we're retrying this
+		   because the final sync_rename_temp_mailbox_node() calls
+		   give different names to local & remote mailbox trees.
+		   something's not right here, but this looping is better than
+		   a crash in sync_mailbox_dirs() due to trees not matching. */
+		goto again;
+	}
 	sync_mailbox_dirs(ctx);
 	return ctx;
 }
diff --git a/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c b/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c
index af416db5546..1defcb2868a 100644
--- a/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c
+++ b/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c
@@ -703,6 +703,28 @@ static void test_dsync_mailbox_tree_sync_renames21(void)
 #endif
 }
 
+static void test_dsync_mailbox_tree_sync_renames22(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 22");
+	tree1 = dsync_mailbox_tree_init('/', '_');
+	tree2 = dsync_mailbox_tree_init('/', '_');
+
+	node_create(tree1, 3, "p/a", 0);
+	node_create(tree1, 0, "p/2", 0);
+	node_create(tree1, 5, "p/2/h", 0);
+
+	node_create(tree2, 4, "p/1/z", 0);
+	node_create(tree2, 1, "p/2", 0);
+	node_create(tree2, 2, "p/2/a", 0);
+	node_create(tree2, 5, "p/2/y", 0);
+	node_create(tree2, 3, "p/3", 0);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
 static void test_dsync_mailbox_tree_sync_random(void)
 {
 	struct dsync_mailbox_tree *tree1, *tree2;
@@ -740,6 +762,7 @@ int main(void)
 		test_dsync_mailbox_tree_sync_renames19,
 		test_dsync_mailbox_tree_sync_renames20,
 		test_dsync_mailbox_tree_sync_renames21,
+		test_dsync_mailbox_tree_sync_renames22,
 		test_dsync_mailbox_tree_sync_random,
 		NULL
 	};

From 55a79cff0ce0d9703fd718a28d0fef2d4a0fcaea Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 22 Apr 2016 18:15:44 +0300
Subject: [PATCH 032/450] lib-mail: Fixed handling duplicate boundary prefixes.

If inner MIME part had the same --boundary prefix as its parent(s) and
the MIME part body started with the inner --boundary prefix, we didn't yet
have it in the list of valid boundaries, so we thought that the outer
boundary was found and the MIME headers were truncated. But due to an extra
bug we still treated it as if it were the inner boundary, except the MIME
part sizes/offsets were set wrong.

This for example fixes a situation where FETCH [1.2.MIME] returns an extra
newline before the actual headers.
---
 src/lib-mail/message-parser.c      |  45 +++++++++--
 src/lib-mail/test-message-parser.c | 116 +++++++++++++++++++++++++++++
 2 files changed, 154 insertions(+), 7 deletions(-)

diff --git a/src/lib-mail/message-parser.c b/src/lib-mail/message-parser.c
index a8177f37857..e936005d567 100644
--- a/src/lib-mail/message-parser.c
+++ b/src/lib-mail/message-parser.c
@@ -45,6 +45,7 @@ struct message_parser_ctx {
 				struct message_block *block_r);
 
 	unsigned int part_seen_content_type:1;
+	unsigned int multipart:1;
 	unsigned int eof:1;
 };
 
@@ -504,6 +505,21 @@ static void parse_content_type(struct message_parser_ctx *ctx,
 	}
 }
 
+static bool block_is_at_eoh(const struct message_block *block)
+{
+	if (block->size < 1)
+		return FALSE;
+	if (block->data[0] == '\n')
+		return TRUE;
+	if (block->data[0] == '\r') {
+		if (block->size < 2)
+			return FALSE;
+		if (block->data[1] == '\n')
+			return TRUE;
+	}
+	return FALSE;
+}
+
 #define MUTEX_FLAGS \
 	(MESSAGE_PART_FLAG_MESSAGE_RFC822 | MESSAGE_PART_FLAG_MULTIPART)
 
@@ -519,12 +535,30 @@ static int parse_next_header(struct message_parser_ctx *ctx,
 	if ((ret = message_parser_read_more(ctx, block_r, &full)) == 0)
 		return ret;
 
+	if (ret > 0 && block_is_at_eoh(block_r) &&
+	    ctx->last_boundary != NULL &&
+	    (part->flags & MESSAGE_PART_FLAG_IS_MIME) != 0) {
+		/* we are at the end of headers and we've determined that we're
+		   going to start a multipart. add the boundary already here
+		   at this point so we can reliably determine whether the
+		   "\n--boundary" belongs to us or to a previous boundary.
+		   this is a problem if the boundary prefixes are identical,
+		   because MIME requires only the prefix to match. */
+		parse_next_body_multipart_init(ctx);
+		ctx->multipart = TRUE;
+	}
+
 	/* before parsing the header see if we can find a --boundary from here.
 	   we're guaranteed to be at the beginning of the line here. */
 	if (ret > 0) {
 		ret = ctx->boundaries == NULL ? -1 :
 			boundary_line_find(ctx, block_r->data,
 					   block_r->size, full, &boundary);
+		if (ret > 0 && boundary->part == ctx->part) {
+			/* our own body begins with our own --boundary.
+			   we don't want to handle that yet. */
+			ret = -1;
+		}
 	}
 	if (ret < 0) {
 		/* no boundary */
@@ -581,14 +615,10 @@ static int parse_next_header(struct message_parser_ctx *ctx,
 	}
 
 	/* end of headers */
-	if ((part->flags & MESSAGE_PART_FLAG_MULTIPART) != 0 &&
-	    ctx->last_boundary == NULL) {
-		/* multipart type but no message boundary */
-		part->flags = 0;
-	}
 	if ((part->flags & MESSAGE_PART_FLAG_IS_MIME) == 0) {
 		/* It's not MIME. Reset everything we found from
 		   Content-Type. */
+		i_assert(!ctx->multipart);
 		part->flags = 0;
 		ctx->last_boundary = NULL;
 	}
@@ -615,8 +645,9 @@ static int parse_next_header(struct message_parser_ctx *ctx,
 	i_assert((part->flags & MUTEX_FLAGS) != MUTEX_FLAGS);
 
 	ctx->last_chr = '\n';
-	if (ctx->last_boundary != NULL) {
-		parse_next_body_multipart_init(ctx);
+	if (ctx->multipart) {
+		i_assert(ctx->last_boundary == NULL);
+		ctx->multipart = FALSE;
 		ctx->parse_next_block = parse_next_body_to_boundary;
 	} else if (part->flags & MESSAGE_PART_FLAG_MESSAGE_RFC822)
 		ctx->parse_next_block = parse_next_body_message_rfc822_init;
diff --git a/src/lib-mail/test-message-parser.c b/src/lib-mail/test-message-parser.c
index da56fb05686..7f7e7b060fe 100644
--- a/src/lib-mail/test-message-parser.c
+++ b/src/lib-mail/test-message-parser.c
@@ -196,6 +196,120 @@ static const char input_msg[] =
 	test_end();
 }
 
+static void test_message_parser_duplicate_mime_boundary(void)
+{
+static const char input_msg[] =
+"Content-Type: multipart/mixed; boundary=\"a\"\n"
+"\n"
+"--a\n"
+"Content-Type: multipart/mixed; boundary=\"a\"\n"
+"\n"
+"--a\n"
+"Content-Type: text/plain\n"
+"\n"
+"body\n";
+	struct message_parser_ctx *parser;
+	struct istream *input;
+	struct message_part *parts;
+	struct message_block block;
+	pool_t pool;
+	int ret;
+
+	test_begin("message parser duplicate mime boundary");
+	pool = pool_alloconly_create("message parser", 10240);
+	input = test_istream_create(input_msg);
+
+	parser = message_parser_init(pool, input, 0, 0);
+	while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ;
+	test_assert(ret < 0);
+	test_assert(message_parser_deinit(&parser, &parts) == 0);
+
+	test_assert(parts->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(parts->header_size.lines == 2);
+	test_assert(parts->header_size.physical_size == 45);
+	test_assert(parts->header_size.virtual_size == 45+2);
+	test_assert(parts->body_size.lines == 7);
+	test_assert(parts->body_size.physical_size == 84);
+	test_assert(parts->body_size.virtual_size == 84+7);
+	test_assert(parts->children->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(parts->children->physical_pos == 49);
+	test_assert(parts->children->header_size.lines == 2);
+	test_assert(parts->children->header_size.physical_size == 45);
+	test_assert(parts->children->header_size.virtual_size == 45+2);
+	test_assert(parts->children->body_size.lines == 4);
+	test_assert(parts->children->body_size.physical_size == 35);
+	test_assert(parts->children->body_size.virtual_size == 35+4);
+	test_assert(parts->children->children->flags == (MESSAGE_PART_FLAG_TEXT | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(parts->children->children->physical_pos == 98);
+	test_assert(parts->children->children->header_size.lines == 2);
+	test_assert(parts->children->children->header_size.physical_size == 26);
+	test_assert(parts->children->children->header_size.virtual_size == 26+2);
+	test_assert(parts->children->children->body_size.lines == 1);
+	test_assert(parts->children->children->body_size.physical_size == 5);
+	test_assert(parts->children->children->body_size.virtual_size == 5+1);
+
+	i_stream_unref(&input);
+	pool_unref(&pool);
+	test_end();
+}
+
+static void test_message_parser_continuing_mime_boundary(void)
+{
+static const char input_msg[] =
+"Content-Type: multipart/mixed; boundary=\"a\"\n"
+"\n"
+"--a\n"
+"Content-Type: multipart/mixed; boundary=\"ab\"\n"
+"\n"
+"--ab\n"
+"Content-Type: text/plain\n"
+"\n"
+"body\n";
+	struct message_parser_ctx *parser;
+	struct istream *input;
+	struct message_part *parts;
+	struct message_block block;
+	pool_t pool;
+	int ret;
+
+	test_begin("message parser continuing mime boundary");
+	pool = pool_alloconly_create("message parser", 10240);
+	input = test_istream_create(input_msg);
+
+	parser = message_parser_init(pool, input, 0, 0);
+	while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ;
+	test_assert(ret < 0);
+	test_assert(message_parser_deinit(&parser, &parts) == 0);
+
+	test_assert(parts->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(parts->header_size.lines == 2);
+	test_assert(parts->header_size.physical_size == 45);
+	test_assert(parts->header_size.virtual_size == 45+2);
+	test_assert(parts->body_size.lines == 7);
+	test_assert(parts->body_size.physical_size == 86);
+	test_assert(parts->body_size.virtual_size == 86+7);
+	test_assert(parts->children->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(parts->children->physical_pos == 49);
+	test_assert(parts->children->header_size.lines == 2);
+	test_assert(parts->children->header_size.physical_size == 46);
+	test_assert(parts->children->header_size.virtual_size == 46+2);
+	test_assert(parts->children->body_size.lines == 4);
+	test_assert(parts->children->body_size.physical_size == 36);
+	test_assert(parts->children->body_size.virtual_size == 36+4);
+	test_assert(parts->children->children->flags == (MESSAGE_PART_FLAG_TEXT | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(parts->children->children->physical_pos == 100);
+	test_assert(parts->children->children->header_size.lines == 2);
+	test_assert(parts->children->children->header_size.physical_size == 26);
+	test_assert(parts->children->children->header_size.virtual_size == 26+2);
+	test_assert(parts->children->children->body_size.lines == 1);
+	test_assert(parts->children->children->body_size.physical_size == 5);
+	test_assert(parts->children->children->body_size.virtual_size == 5+1);
+
+	i_stream_unref(&input);
+	pool_unref(&pool);
+	test_end();
+}
+
 static void test_message_parser_no_eoh(void)
 {
 	static const char input_msg[] = "a:b\n";
@@ -228,6 +342,8 @@ int main(void)
 	static void (*test_functions[])(void) = {
 		test_message_parser_small_blocks,
 		test_message_parser_truncated_mime_headers,
+		test_message_parser_duplicate_mime_boundary,
+		test_message_parser_continuing_mime_boundary,
 		test_message_parser_no_eoh,
 		NULL
 	};

From 80622816d4f381c01b764f38cd32a34c7e7f8925 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 22 Apr 2016 18:28:20 +0300
Subject: [PATCH 033/450] lib-mail: Fixed inner MIME part boundary being a
 prefix of outer boundary.

---
 src/lib-mail/message-parser.c      | 12 ++--
 src/lib-mail/test-message-parser.c | 96 ++++++++++++++++++++++++++++++
 2 files changed, 104 insertions(+), 4 deletions(-)

diff --git a/src/lib-mail/message-parser.c b/src/lib-mail/message-parser.c
index e936005d567..e3e712a298d 100644
--- a/src/lib-mail/message-parser.c
+++ b/src/lib-mail/message-parser.c
@@ -66,19 +66,23 @@ static struct message_boundary *
 boundary_find(struct message_boundary *boundaries,
 	      const unsigned char *data, size_t len)
 {
+	struct message_boundary *best = NULL;
+
 	/* As MIME spec says: search from latest one to oldest one so that we
 	   don't break if the same boundary is used in nested parts. Also the
 	   full message line doesn't have to match the boundary, only the
-	   beginning. */
+	   beginning. However, if there are multiple prefixes whose beginning
+	   matches, use the longest matching one. */
 	while (boundaries != NULL) {
 		if (boundaries->len <= len &&
-		    memcmp(boundaries->boundary, data, boundaries->len) == 0)
-			return boundaries;
+		    memcmp(boundaries->boundary, data, boundaries->len) == 0 &&
+		    (best == NULL || best->len < boundaries->len))
+			best = boundaries;
 
 		boundaries = boundaries->next;
 	}
 
-	return NULL;
+	return best;
 }
 
 static void parse_body_add_block(struct message_parser_ctx *ctx,
diff --git a/src/lib-mail/test-message-parser.c b/src/lib-mail/test-message-parser.c
index 7f7e7b060fe..c613bb53064 100644
--- a/src/lib-mail/test-message-parser.c
+++ b/src/lib-mail/test-message-parser.c
@@ -196,6 +196,100 @@ static const char input_msg[] =
 	test_end();
 }
 
+static void test_message_parser_truncated_mime_headers2(void)
+{
+static const char input_msg[] =
+"Content-Type: multipart/mixed; boundary=\"ab\"\n"
+"\n"
+"--ab\n"
+"Content-Type: multipart/mixed; boundary=\"a\"\n"
+"\n"
+"--ab\n"
+"Content-Type: text/plain\n"
+"\n"
+"--a\n\n";
+	struct message_parser_ctx *parser;
+	struct istream *input;
+	struct message_part *parts;
+	struct message_block block;
+	pool_t pool;
+	int ret;
+
+	test_begin("message parser truncated mime headers 2");
+	pool = pool_alloconly_create("message parser", 10240);
+	input = test_istream_create(input_msg);
+
+	parser = message_parser_init(pool, input, 0, 0);
+	while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ;
+	test_assert(ret < 0);
+	test_assert(message_parser_deinit(&parser, &parts) == 0);
+
+	test_assert(parts->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(parts->header_size.lines == 2);
+	test_assert(parts->header_size.physical_size == 46);
+	test_assert(parts->header_size.virtual_size == 46+2);
+	test_assert(parts->body_size.lines == 8);
+	test_assert(parts->body_size.physical_size == 86);
+	test_assert(parts->body_size.virtual_size == 86+8);
+
+	test_assert(parts->children->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(parts->children->physical_pos == 51);
+	test_assert(parts->children->header_size.lines == 1);
+	test_assert(parts->children->header_size.physical_size == 44);
+	test_assert(parts->children->header_size.virtual_size == 44+1);
+	test_assert(parts->children->body_size.lines == 0);
+	test_assert(parts->children->body_size.physical_size == 0);
+	test_assert(parts->children->children == NULL);
+
+	test_assert(parts->children->next->flags == (MESSAGE_PART_FLAG_TEXT | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(parts->children->next->physical_pos == 101);
+	test_assert(parts->children->next->header_size.lines == 2);
+	test_assert(parts->children->next->header_size.physical_size == 26);
+	test_assert(parts->children->next->header_size.virtual_size == 26+2);
+	test_assert(parts->children->next->body_size.lines == 2);
+	test_assert(parts->children->next->body_size.physical_size == 5);
+	test_assert(parts->children->next->body_size.virtual_size == 5+2);
+	test_assert(parts->children->next->children == NULL);
+
+	i_stream_unref(&input);
+	pool_unref(&pool);
+	test_end();
+}
+
+static void test_message_parser_truncated_mime_headers3(void)
+{
+static const char input_msg[] =
+"Content-Type: multipart/mixed; boundary=\"ab\"\n";
+	struct message_parser_ctx *parser;
+	struct istream *input;
+	struct message_part *parts;
+	struct message_block block;
+	pool_t pool;
+	int ret;
+
+	test_begin("message parser truncated mime headers 3");
+	pool = pool_alloconly_create("message parser", 10240);
+	input = test_istream_create(input_msg);
+
+	parser = message_parser_init(pool, input, 0, 0);
+	while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ;
+	test_assert(ret < 0);
+	test_assert(message_parser_deinit(&parser, &parts) == 0);
+
+	test_assert(parts->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(parts->header_size.lines == 1);
+	test_assert(parts->header_size.physical_size == 45);
+	test_assert(parts->header_size.virtual_size == 45+1);
+	test_assert(parts->body_size.lines == 0);
+	test_assert(parts->body_size.physical_size == 0);
+
+	test_assert(parts->children == NULL);
+
+	i_stream_unref(&input);
+	pool_unref(&pool);
+	test_end();
+}
+
 static void test_message_parser_duplicate_mime_boundary(void)
 {
 static const char input_msg[] =
@@ -342,6 +436,8 @@ int main(void)
 	static void (*test_functions[])(void) = {
 		test_message_parser_small_blocks,
 		test_message_parser_truncated_mime_headers,
+		test_message_parser_truncated_mime_headers2,
+		test_message_parser_truncated_mime_headers3,
 		test_message_parser_duplicate_mime_boundary,
 		test_message_parser_continuing_mime_boundary,
 		test_message_parser_no_eoh,

From c6f44125b64cb29ed767fefdcb4c08ff4efe299d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 22 Apr 2016 18:35:04 +0300
Subject: [PATCH 034/450] lib-mail: Improved test-message-parser further

---
 src/lib-mail/test-message-parser.c | 96 ++++++++++++++++++++++++++++++
 1 file changed, 96 insertions(+)

diff --git a/src/lib-mail/test-message-parser.c b/src/lib-mail/test-message-parser.c
index c613bb53064..5df4b1387b2 100644
--- a/src/lib-mail/test-message-parser.c
+++ b/src/lib-mail/test-message-parser.c
@@ -290,6 +290,43 @@ static const char input_msg[] =
 	test_end();
 }
 
+static void test_message_parser_empty_multipart(void)
+{
+static const char input_msg[] =
+"Content-Type: multipart/mixed; boundary=\"ab\"\n"
+"\n"
+"body\n";
+	struct message_parser_ctx *parser;
+	struct istream *input;
+	struct message_part *parts;
+	struct message_block block;
+	pool_t pool;
+	int ret;
+
+	test_begin("message parser truncated mime headers 3");
+	pool = pool_alloconly_create("message parser", 10240);
+	input = test_istream_create(input_msg);
+
+	parser = message_parser_init(pool, input, 0, 0);
+	while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ;
+	test_assert(ret < 0);
+	test_assert(message_parser_deinit(&parser, &parts) == 0);
+
+	test_assert(parts->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(parts->header_size.lines == 2);
+	test_assert(parts->header_size.physical_size == 46);
+	test_assert(parts->header_size.virtual_size == 46+2);
+	test_assert(parts->body_size.lines == 1);
+	test_assert(parts->body_size.physical_size == 5);
+	test_assert(parts->body_size.virtual_size == 5+1);
+
+	test_assert(parts->children == NULL);
+
+	i_stream_unref(&input);
+	pool_unref(&pool);
+	test_end();
+}
+
 static void test_message_parser_duplicate_mime_boundary(void)
 {
 static const char input_msg[] =
@@ -347,6 +384,63 @@ static const char input_msg[] =
 	test_end();
 }
 
+static void test_message_parser_garbage_suffix_mime_boundary(void)
+{
+static const char input_msg[] =
+"Content-Type: multipart/mixed; boundary=\"a\"\n"
+"\n"
+"--ab\n"
+"Content-Type: multipart/mixed; boundary=\"a\"\n"
+"\n"
+"--ac\n"
+"Content-Type: text/plain\n"
+"\n"
+"body\n";
+	struct message_parser_ctx *parser;
+	struct istream *input;
+	struct message_part *parts;
+	struct message_block block;
+	pool_t pool;
+	int ret;
+
+	test_begin("message parser garbage suffix mime boundary");
+	pool = pool_alloconly_create("message parser", 10240);
+	input = test_istream_create(input_msg);
+
+	parser = message_parser_init(pool, input, 0, 0);
+	while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ;
+	test_assert(ret < 0);
+	test_assert(message_parser_deinit(&parser, &parts) == 0);
+
+	test_assert(parts->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(parts->header_size.lines == 2);
+	test_assert(parts->header_size.physical_size == 45);
+	test_assert(parts->header_size.virtual_size == 45+2);
+	test_assert(parts->body_size.lines == 7);
+	test_assert(parts->body_size.physical_size == 86);
+	test_assert(parts->body_size.virtual_size == 86+7);
+	test_assert(parts->children->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(parts->children->physical_pos == 50);
+	test_assert(parts->children->header_size.lines == 2);
+	test_assert(parts->children->header_size.physical_size == 45);
+	test_assert(parts->children->header_size.virtual_size == 45+2);
+	test_assert(parts->children->body_size.lines == 4);
+	test_assert(parts->children->body_size.physical_size == 36);
+	test_assert(parts->children->body_size.virtual_size == 36+4);
+	test_assert(parts->children->children->flags == (MESSAGE_PART_FLAG_TEXT | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(parts->children->children->physical_pos == 100);
+	test_assert(parts->children->children->header_size.lines == 2);
+	test_assert(parts->children->children->header_size.physical_size == 26);
+	test_assert(parts->children->children->header_size.virtual_size == 26+2);
+	test_assert(parts->children->children->body_size.lines == 1);
+	test_assert(parts->children->children->body_size.physical_size == 5);
+	test_assert(parts->children->children->body_size.virtual_size == 5+1);
+
+	i_stream_unref(&input);
+	pool_unref(&pool);
+	test_end();
+}
+
 static void test_message_parser_continuing_mime_boundary(void)
 {
 static const char input_msg[] =
@@ -438,7 +532,9 @@ int main(void)
 		test_message_parser_truncated_mime_headers,
 		test_message_parser_truncated_mime_headers2,
 		test_message_parser_truncated_mime_headers3,
+		test_message_parser_empty_multipart,
 		test_message_parser_duplicate_mime_boundary,
+		test_message_parser_garbage_suffix_mime_boundary,
 		test_message_parser_continuing_mime_boundary,
 		test_message_parser_no_eoh,
 		NULL

From ed2e2ac2629955c12a3a8037412a7465266479ea Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 19 Apr 2016 19:07:51 +0300
Subject: [PATCH 035/450] doveadm user: Code cleanup - move field printing to
 its own function.

No functional changes.
---
 src/doveadm/doveadm-auth.c | 74 ++++++++++++++++++++++----------------
 1 file changed, 43 insertions(+), 31 deletions(-)

diff --git a/src/doveadm/doveadm-auth.c b/src/doveadm/doveadm-auth.c
index 55aa9b15533..d1dcdaa0726 100644
--- a/src/doveadm/doveadm-auth.c
+++ b/src/doveadm/doveadm-auth.c
@@ -489,16 +489,52 @@ static void cmd_user_mail_input_field(const char *key, const char *value,
 	}
 }
 
-static int cmd_user_mail_input(struct mail_storage_service_ctx *storage_service,
-			       const struct authtest_input *input,
-			       const char *show_field)
+static void
+cmd_user_mail_print_fields(const struct authtest_input *input,
+			   struct mail_user *user,
+			   const char *const *userdb_fields,
+			   const char *show_field)
+{
+	const struct mail_storage_settings *mail_set;
+	const char *key, *value;
+	unsigned int i;
+
+	if (strcmp(input->username, user->username) != 0)
+		cmd_user_mail_input_field("user", user->username, show_field);
+	cmd_user_mail_input_field("uid", user->set->mail_uid, show_field);
+	cmd_user_mail_input_field("gid", user->set->mail_gid, show_field);
+	cmd_user_mail_input_field("home", user->set->mail_home, show_field);
+
+	mail_set = mail_user_set_get_storage_set(user);
+	cmd_user_mail_input_field("mail", mail_set->mail_location, show_field);
+
+	if (userdb_fields != NULL) {
+		for (i = 0; userdb_fields[i] != NULL; i++) {
+			value = strchr(userdb_fields[i], '=');
+			if (value != NULL)
+				key = t_strdup_until(userdb_fields[i], value++);
+			else {
+				key = userdb_fields[i];
+				value = "";
+			}
+			if (strcmp(key, "uid") != 0 &&
+			    strcmp(key, "gid") != 0 &&
+			    strcmp(key, "home") != 0 &&
+			    strcmp(key, "mail") != 0)
+				cmd_user_mail_input_field(key, value, show_field);
+		}
+	}
+}
+
+static int
+cmd_user_mail_input(struct mail_storage_service_ctx *storage_service,
+		    const struct authtest_input *input,
+		    const char *show_field)
 {
 	struct mail_storage_service_input service_input;
 	struct mail_storage_service_user *service_user;
 	struct mail_user *user;
-	const struct mail_storage_settings *mail_set;
-	const char *key, *value, *error, *const *userdb_fields;
-	unsigned int i;
+	const char *error, *const *userdb_fields;
 	pool_t pool;
 	int ret;
 
@@ -528,31 +564,7 @@ static int cmd_user_mail_input(struct mail_storage_service_ctx *storage_service,
 		return 0;
 	}
 
-	if (strcmp(input->username, user->username) != 0)
-		cmd_user_mail_input_field("user", user->username, show_field);
-	cmd_user_mail_input_field("uid", user->set->mail_uid, show_field);
-	cmd_user_mail_input_field("gid", user->set->mail_gid, show_field);
-	cmd_user_mail_input_field("home", user->set->mail_home, show_field);
-
-	mail_set = mail_user_set_get_storage_set(user);
-	cmd_user_mail_input_field("mail", mail_set->mail_location, show_field);
-
-	if (userdb_fields != NULL) {
-		for (i = 0; userdb_fields[i] != NULL; i++) {
-			value = strchr(userdb_fields[i], '=');
-			if (value != NULL)
-				key = t_strdup_until(userdb_fields[i], value++);
-			else {
-				key = userdb_fields[i];
-				value = "";
-			}
-			if (strcmp(key, "uid") != 0 &&
-			    strcmp(key, "gid") != 0 &&
-			    strcmp(key, "home") != 0 &&
-			    strcmp(key, "mail") != 0)
-				cmd_user_mail_input_field(key, value, show_field);
-		}
-	}
+	cmd_user_mail_print_fields(input, user, userdb_fields, show_field);
 
 	mail_user_unref(&user);
 	mail_storage_service_user_free(&service_user);

From cfb90f18ea813be72db99a8e8fe9bfa4dcd50976 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 19 Apr 2016 19:08:52 +0300
Subject: [PATCH 036/450] doveadm user: Added -e <value> parameter to expand
 %variables in <value>

This allows expanding any %variable that the mail processes normally expand.
For example:

% doveadm user -e "%u's home is %h" testuser@example.com
testuser@example.com's home is /var/mail/testuser@example.com
---
 src/doveadm/doveadm-auth.c | 39 ++++++++++++++++++++++++++++++--------
 1 file changed, 31 insertions(+), 8 deletions(-)

diff --git a/src/doveadm/doveadm-auth.c b/src/doveadm/doveadm-auth.c
index d1dcdaa0726..338deb6d289 100644
--- a/src/doveadm/doveadm-auth.c
+++ b/src/doveadm/doveadm-auth.c
@@ -7,6 +7,7 @@
 #include "base64.h"
 #include "hex-binary.h"
 #include "str.h"
+#include "var-expand.h"
 #include "wildcard-match.h"
 #include "settings-parser.h"
 #include "master-service.h"
@@ -529,7 +530,7 @@ cmd_user_mail_print_fields(const struct authtest_input *input,
 static int
 cmd_user_mail_input(struct mail_storage_service_ctx *storage_service,
 		    const struct authtest_input *input,
-		    const char *show_field)
+		    const char *show_field, const char *expand_field)
 {
 	struct mail_storage_service_input service_input;
 	struct mail_storage_service_user *service_user;
@@ -558,13 +559,21 @@ cmd_user_mail_input(struct mail_storage_service_ctx *storage_service,
 		pool_unref(&pool);
 		if (ret < 0)
 			return -1;
-		fprintf(show_field == NULL ? stdout : stderr,
+		fprintf(show_field == NULL && expand_field == NULL ? stdout : stderr,
 			"userdb lookup: user %s doesn't exist\n",
 			input->username);
 		return 0;
 	}
 
-	cmd_user_mail_print_fields(input, user, userdb_fields, show_field);
+	if (expand_field == NULL)
+		cmd_user_mail_print_fields(input, user, userdb_fields, show_field);
+	else {
+		string_t *str = t_str_new(128);
+		var_expand_with_funcs(str, expand_field,
+				      mail_user_var_expand_table(user),
+				      mail_user_var_expand_func_table, user);
+		printf("%s\n", str_c(str));
+	}
 
 	mail_user_unref(&user);
 	mail_storage_service_user_free(&service_user);
@@ -577,18 +586,21 @@ static void cmd_user(int argc, char *argv[])
 	const char *auth_socket_path = doveadm_settings->auth_socket_path;
 	struct auth_master_connection *conn;
 	struct authtest_input input;
-	const char *show_field = NULL;
+	const char *show_field = NULL, *expand_field = NULL;
 	struct mail_storage_service_ctx *storage_service = NULL;
 	unsigned int i;
 	bool have_wildcards, userdb_only = FALSE, first = TRUE;
 	int c, ret;
 
 	authtest_input_init(&input);
-	while ((c = getopt(argc, argv, "a:f:ux:")) > 0) {
+	while ((c = getopt(argc, argv, "a:e:f:ux:")) > 0) {
 		switch (c) {
 		case 'a':
 			auth_socket_path = optarg;
 			break;
+		case 'e':
+			expand_field = optarg;
+			break;
 		case 'f':
 			show_field = optarg;
 			break;
@@ -603,6 +615,17 @@ static void cmd_user(int argc, char *argv[])
 		}
 	}
 
+	if (expand_field != NULL && userdb_only) {
+		i_error("-e can't be used with -u");
+		doveadm_exit_code = EX_USAGE;
+		return;
+	}
+	if (expand_field != NULL && show_field != NULL) {
+		i_error("-e can't be used with -f");
+		doveadm_exit_code = EX_USAGE;
+		return;
+	}
+
 	if (optind == argc)
 		auth_cmd_help(cmd_user);
 
@@ -633,7 +656,7 @@ static void cmd_user(int argc, char *argv[])
 			MAIL_STORAGE_SERVICE_FLAG_NO_RESTRICT_ACCESS);
 		mail_storage_service_set_auth_conn(storage_service, conn);
 		conn = NULL;
-		if (show_field == NULL) {
+		if (show_field == NULL && expand_field == NULL) {
 			doveadm_print_init(DOVEADM_PRINT_TYPE_TAB);
 			doveadm_print_header_simple("field");
 			doveadm_print_header_simple("value");
@@ -647,7 +670,7 @@ static void cmd_user(int argc, char *argv[])
 			putchar('\n');
 
 		ret = !userdb_only ?
-			cmd_user_mail_input(storage_service, &input, show_field) :
+			cmd_user_mail_input(storage_service, &input, show_field, expand_field) :
 			cmd_user_input(conn, &input, show_field, TRUE);
 		switch (ret) {
 		case -1:
@@ -674,7 +697,7 @@ struct doveadm_cmd doveadm_cmd_auth[] = {
 	{ cmd_auth_cache_flush, "auth cache flush",
 	  "[-a <master socket path>] [<user> [...]]" },
 	{ cmd_user, "user",
-	  "[-a <userdb socket path>] [-x <auth info>] [-f field] [-u] <user mask> [...]" }
+	  "[-a <userdb socket path>] [-x <auth info>] [-f field] [-e <value>] [-u] <user mask> [...]" }
 };
 
 static void auth_cmd_help(doveadm_command_t *cmd)

From bfbb24b9faba009c2cb79b3a38995acf2621f49b Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 21 Apr 2016 18:51:57 +0300
Subject: [PATCH 037/450] Split parts of lib-dict into lib-dict-extra.

Otherwise there's a circular dependency because lib-dict/dict-fs.c depends
on lib-fs, while lib-fs/fs-dict.c depends on lib-dict. This becomes a problem
when compiling --without-shared-libs, although for some reason it works for
me while linking the Dovecot core, but not when linking external plugins.
---
 configure.ac                                     |  3 ++-
 src/Makefile.am                                  |  1 +
 src/lib-dict-extra/Makefile.am                   | 14 ++++++++++++++
 src/{lib-dict => lib-dict-extra}/dict-fs.c       |  0
 src/{lib-dict => lib-dict-extra}/dict-register.c |  0
 src/lib-dict/Makefile.am                         |  5 -----
 src/lib-dovecot/Makefile.am                      |  1 +
 7 files changed, 18 insertions(+), 6 deletions(-)
 create mode 100644 src/lib-dict-extra/Makefile.am
 rename src/{lib-dict => lib-dict-extra}/dict-fs.c (100%)
 rename src/{lib-dict => lib-dict-extra}/dict-register.c (100%)

diff --git a/configure.ac b/configure.ac
index 84297442e5a..821451603b6 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2523,7 +2523,7 @@ if test "$want_shared_libs" = "yes"; then
   LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/libdovecot-login.la'
   LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/libdovecot-lda.la'
 else
-  LIBDOVECOT_DEPS='$(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-stats/libstats.la $(top_builddir)/src/lib-http/libhttp.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-sasl/libsasl.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la $(top_builddir)/src/lib-test/libtest.la $(top_builddir)/src/lib/liblib.la'
+  LIBDOVECOT_DEPS='$(top_builddir)/src/lib-dict-extra/libdict_extra.la $(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-stats/libstats.la $(top_builddir)/src/lib-http/libhttp.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-sasl/libsasl.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la $(top_builddir)/src/lib-test/libtest.la $(top_builddir)/src/lib/liblib.la'
   LIBDOVECOT="$LIBDOVECOT_DEPS \$(LIBICONV) \$(MODULE_LIBS)"
   LIBDOVECOT_STORAGE_DEPS='$(top_builddir)/src/lib-storage/libstorage.la'
   LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/liblogin.la'
@@ -2859,6 +2859,7 @@ src/lib-auth/Makefile
 src/lib-charset/Makefile
 src/lib-compression/Makefile
 src/lib-dict/Makefile
+src/lib-dict-extra/Makefile
 src/lib-dns/Makefile
 src/lib-fs/Makefile
 src/lib-fts/Makefile
diff --git a/src/Makefile.am b/src/Makefile.am
index cafcef86df1..87501de2793 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -23,6 +23,7 @@ LIBDOVECOT_SUBDIRS = \
 
 SUBDIRS = \
 	$(LIBDOVECOT_SUBDIRS) \
+	lib-dict-extra \
 	lib-dovecot \
 	lib-fts \
 	lib-imap-client \
diff --git a/src/lib-dict-extra/Makefile.am b/src/lib-dict-extra/Makefile.am
new file mode 100644
index 00000000000..c292ba23361
--- /dev/null
+++ b/src/lib-dict-extra/Makefile.am
@@ -0,0 +1,14 @@
+noinst_LTLIBRARIES = libdict_extra.la
+
+dict_drivers = @dict_drivers@
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-dict \
+	-I$(top_srcdir)/src/lib-fs \
+	-I$(top_srcdir)/src/lib-settings \
+	$(SQL_CFLAGS)
+
+libdict_extra_la_SOURCES = \
+	dict-fs.c \
+	dict-register.c
diff --git a/src/lib-dict/dict-fs.c b/src/lib-dict-extra/dict-fs.c
similarity index 100%
rename from src/lib-dict/dict-fs.c
rename to src/lib-dict-extra/dict-fs.c
diff --git a/src/lib-dict/dict-register.c b/src/lib-dict-extra/dict-register.c
similarity index 100%
rename from src/lib-dict/dict-register.c
rename to src/lib-dict-extra/dict-register.c
diff --git a/src/lib-dict/Makefile.am b/src/lib-dict/Makefile.am
index bf893b0f886..07fbda079e4 100644
--- a/src/lib-dict/Makefile.am
+++ b/src/lib-dict/Makefile.am
@@ -1,12 +1,9 @@
 noinst_LTLIBRARIES = libdict.la
 noinst_LIBRARIES = libdict_backend.a
 
-dict_drivers = @dict_drivers@
-
 AM_CPPFLAGS = \
 	-I$(top_srcdir)/src/lib \
 	-I$(top_srcdir)/src/lib-test \
-	-I$(top_srcdir)/src/lib-fs \
 	-I$(top_srcdir)/src/lib-ldap \
 	-I$(top_srcdir)/src/lib-sql \
 	-I$(top_srcdir)/src/lib-settings \
@@ -16,11 +13,9 @@ base_sources = \
 	dict.c \
 	dict-client.c \
 	dict-file.c \
-	dict-fs.c \
 	dict-memcached.c \
 	dict-memcached-ascii.c \
 	dict-redis.c \
-	dict-register.c \
 	dict-transaction-memory.c
 
 libdict_la_SOURCES = \
diff --git a/src/lib-dovecot/Makefile.am b/src/lib-dovecot/Makefile.am
index fd087760869..3279f89dd9e 100644
--- a/src/lib-dovecot/Makefile.am
+++ b/src/lib-dovecot/Makefile.am
@@ -1,6 +1,7 @@
 # when adding libraries, update LIBDOVECOT also in configure.in
 libs = \
 	../lib-master/libmaster.la \
+	../lib-dict-extra/libdict_extra.la \
 	../lib-fs/libfs.la \
 	../lib-settings/libsettings.la \
 	../lib-stats/libstats.la \

From 2a81294e43c85d034acfb897e2532b38b9c06e7c Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 21 Apr 2016 18:58:10 +0300
Subject: [PATCH 038/450] lib-dict: Moved dict-ldap to lib-dict-extra

This also allows moving lib-ldap away from LIBDOVECOT_SUBDIRS in
src/Makefile.am, which was wrong because it's not really part of
libdovecot.la.
---
 src/Makefile.am                               |  2 +-
 src/lib-dict-extra/Makefile.am                | 22 +++++++++++++++++--
 .../dict-ldap-settings.c                      |  0
 .../dict-ldap-settings.h                      |  0
 src/{lib-dict => lib-dict-extra}/dict-ldap.c  |  0
 src/lib-dict/Makefile.am                      | 17 --------------
 6 files changed, 21 insertions(+), 20 deletions(-)
 rename src/{lib-dict => lib-dict-extra}/dict-ldap-settings.c (100%)
 rename src/{lib-dict => lib-dict-extra}/dict-ldap-settings.h (100%)
 rename src/{lib-dict => lib-dict-extra}/dict-ldap.c (100%)

diff --git a/src/Makefile.am b/src/Makefile.am
index 87501de2793..39c62d213a7 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -10,7 +10,6 @@ LIBDOVECOT_SUBDIRS = \
 	lib-master \
 	lib-charset \
 	lib-dns \
-	$(LIB_LDAP) \
 	lib-dict \
 	lib-sasl \
 	lib-ssl-iostream \
@@ -23,6 +22,7 @@ LIBDOVECOT_SUBDIRS = \
 
 SUBDIRS = \
 	$(LIBDOVECOT_SUBDIRS) \
+	$(LIB_LDAP) \
 	lib-dict-extra \
 	lib-dovecot \
 	lib-fts \
diff --git a/src/lib-dict-extra/Makefile.am b/src/lib-dict-extra/Makefile.am
index c292ba23361..7cf50114a06 100644
--- a/src/lib-dict-extra/Makefile.am
+++ b/src/lib-dict-extra/Makefile.am
@@ -6,9 +6,27 @@ AM_CPPFLAGS = \
 	-I$(top_srcdir)/src/lib \
 	-I$(top_srcdir)/src/lib-dict \
 	-I$(top_srcdir)/src/lib-fs \
-	-I$(top_srcdir)/src/lib-settings \
-	$(SQL_CFLAGS)
+	-I$(top_srcdir)/src/lib-ldap \
+	-I$(top_srcdir)/src/lib-settings
 
 libdict_extra_la_SOURCES = \
 	dict-fs.c \
 	dict-register.c
+
+NOPLUGIN_LDFLAGS =
+
+if HAVE_LDAP
+LIBDICT_LDAP = libdict_ldap.la
+endif
+libdict_ldap_la_LDFLAGS = -module -avoid-version $(LIBDOVECOT_LDAP)
+
+module_dictdir = $(moduledir)/dict
+module_dict_LTLIBRARIES = \
+	$(LIBDICT_LDAP)
+
+libdict_ldap_la_SOURCES = \
+	dict-ldap.c \
+	dict-ldap-settings.c
+
+noinst_HEADERS = \
+	dict-ldap-settings.h
diff --git a/src/lib-dict/dict-ldap-settings.c b/src/lib-dict-extra/dict-ldap-settings.c
similarity index 100%
rename from src/lib-dict/dict-ldap-settings.c
rename to src/lib-dict-extra/dict-ldap-settings.c
diff --git a/src/lib-dict/dict-ldap-settings.h b/src/lib-dict-extra/dict-ldap-settings.h
similarity index 100%
rename from src/lib-dict/dict-ldap-settings.h
rename to src/lib-dict-extra/dict-ldap-settings.h
diff --git a/src/lib-dict/dict-ldap.c b/src/lib-dict-extra/dict-ldap.c
similarity index 100%
rename from src/lib-dict/dict-ldap.c
rename to src/lib-dict-extra/dict-ldap.c
diff --git a/src/lib-dict/Makefile.am b/src/lib-dict/Makefile.am
index 07fbda079e4..805ecc6a7cb 100644
--- a/src/lib-dict/Makefile.am
+++ b/src/lib-dict/Makefile.am
@@ -4,7 +4,6 @@ noinst_LIBRARIES = libdict_backend.a
 AM_CPPFLAGS = \
 	-I$(top_srcdir)/src/lib \
 	-I$(top_srcdir)/src/lib-test \
-	-I$(top_srcdir)/src/lib-ldap \
 	-I$(top_srcdir)/src/lib-sql \
 	-I$(top_srcdir)/src/lib-settings \
 	$(SQL_CFLAGS)
@@ -30,25 +29,9 @@ libdict_backend_a_SOURCES = \
 nodist_libdict_backend_a_SOURCES = \
 	dict-drivers-register.c
 
-NOPLUGIN_LDFLAGS =
-
-if HAVE_LDAP
-LIBDICT_LDAP = libdict_ldap.la
-endif
-libdict_ldap_la_LDFLAGS = -module -avoid-version $(LIBDOVECOT_LDAP)
-
-module_dictdir = $(moduledir)/dict
-module_dict_LTLIBRARIES = \
-	$(LIBDICT_LDAP)
-
-libdict_ldap_la_SOURCES = \
-	dict-ldap.c \
-	dict-ldap-settings.c
-
 headers = \
 	dict.h \
 	dict-client.h \
-	dict-ldap-settings.h \
 	dict-private.h \
 	dict-sql.h \
 	dict-sql-settings.h \

From 4968bfaf02745ee8215e03d1effbc95bfa6cdeda Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 21 Apr 2016 19:18:55 +0300
Subject: [PATCH 039/450] lib-dovecot: Avoid duplication of listing *.la in
 both Makefile.am and configure

---
 configure.ac                |  4 +++-
 src/lib-dovecot/Makefile.am | 23 ++---------------------
 2 files changed, 5 insertions(+), 22 deletions(-)

diff --git a/configure.ac b/configure.ac
index 821451603b6..95259a2510f 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2516,6 +2516,7 @@ dnl **
 dnl ** Shared libraries usage
 dnl **
 
+LIBDOVECOT_LA_LIBS='$(top_builddir)/src/lib-dict-extra/libdict_extra.la $(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-stats/libstats.la $(top_builddir)/src/lib-http/libhttp.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-sasl/libsasl.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la $(top_builddir)/src/lib-test/libtest.la $(top_builddir)/src/lib/liblib.la'
 if test "$want_shared_libs" = "yes"; then
   LIBDOVECOT_DEPS='$(top_builddir)/src/lib-dovecot/libdovecot.la'
   LIBDOVECOT="$LIBDOVECOT_DEPS \$(MODULE_LIBS)"
@@ -2523,7 +2524,7 @@ if test "$want_shared_libs" = "yes"; then
   LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/libdovecot-login.la'
   LIBDOVECOT_LDA='$(top_builddir)/src/lib-lda/libdovecot-lda.la'
 else
-  LIBDOVECOT_DEPS='$(top_builddir)/src/lib-dict-extra/libdict_extra.la $(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-stats/libstats.la $(top_builddir)/src/lib-http/libhttp.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-sasl/libsasl.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la $(top_builddir)/src/lib-test/libtest.la $(top_builddir)/src/lib/liblib.la'
+  LIBDOVECOT_DEPS="$LIBDOVECOT_LA_LIBS"
   LIBDOVECOT="$LIBDOVECOT_DEPS \$(LIBICONV) \$(MODULE_LIBS)"
   LIBDOVECOT_STORAGE_DEPS='$(top_builddir)/src/lib-storage/libstorage.la'
   LIBDOVECOT_LOGIN='$(top_builddir)/src/login-common/liblogin.la'
@@ -2540,6 +2541,7 @@ LIBDOVECOT_SQL='$(top_builddir)/src/lib-sql/libsql.la'
 LIBDOVECOT_COMPRESS='$(top_builddir)/src/lib-compression/libcompression.la'
 LIBDOVECOT_LIBFTS='$(top_builddir)/src/lib-fts/libfts.la'
 AC_SUBST(LIBDOVECOT)
+AC_SUBST(LIBDOVECOT_LA_LIBS)
 AC_SUBST(LIBDOVECOT_DEPS)
 AC_SUBST(LIBDOVECOT_STORAGE)
 AC_SUBST(LIBDOVECOT_STORAGE_DEPS)
diff --git a/src/lib-dovecot/Makefile.am b/src/lib-dovecot/Makefile.am
index 3279f89dd9e..a8820e5e197 100644
--- a/src/lib-dovecot/Makefile.am
+++ b/src/lib-dovecot/Makefile.am
@@ -1,29 +1,10 @@
-# when adding libraries, update LIBDOVECOT also in configure.in
-libs = \
-	../lib-master/libmaster.la \
-	../lib-dict-extra/libdict_extra.la \
-	../lib-fs/libfs.la \
-	../lib-settings/libsettings.la \
-	../lib-stats/libstats.la \
-	../lib-http/libhttp.la \
-	../lib-dict/libdict.la \
-	../lib-imap/libimap.la \
-	../lib-mail/libmail.la \
-	../lib-sasl/libsasl.la \
-	../lib-auth/libauth.la \
-	../lib-dns/libdns.la \
-	../lib-charset/libcharset.la \
-	../lib-ssl-iostream/libssl_iostream.la \
-	../lib-test/libtest.la \
-	../lib/liblib.la
-
 pkglib_LTLIBRARIES = libdovecot.la
 
 libdovecot_la_SOURCES = 
 
 libdovecot_la_LIBADD = \
-	$(libs) \
+	$(LIBDOVECOT_LA_LIBS) \
 	$(MODULE_LIBS)
 
-libdovecot_la_DEPENDENCIES = $(libs)
+libdovecot_la_DEPENDENCIES = $(LIBDOVECOT_LA_LIBS)
 libdovecot_la_LDFLAGS = -export-dynamic

From 0bbaf397a41a89d35f795b51d6e871613712b006 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 22 Apr 2016 19:37:57 +0300
Subject: [PATCH 040/450] cassandra: Added num_threads, connect_timeout and
 request_timeout settings.

---
 src/lib-sql/Makefile.am        |  1 +
 src/lib-sql/driver-cassandra.c | 22 +++++++++++++++++++---
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/src/lib-sql/Makefile.am b/src/lib-sql/Makefile.am
index 489f257beeb..a834a08180f 100644
--- a/src/lib-sql/Makefile.am
+++ b/src/lib-sql/Makefile.am
@@ -36,6 +36,7 @@ sql_drivers = @sql_drivers@
 
 AM_CPPFLAGS = \
 	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-settings \
 	$(SQL_CFLAGS)
 
 dist_sources = \
diff --git a/src/lib-sql/driver-cassandra.c b/src/lib-sql/driver-cassandra.c
index 231e72328b4..13f46f45726 100644
--- a/src/lib-sql/driver-cassandra.c
+++ b/src/lib-sql/driver-cassandra.c
@@ -8,6 +8,7 @@
 #include "net.h"
 #include "write-full.h"
 #include "time-util.h"
+#include "settings-parser.h"
 #include "sql-api-private.h"
 
 #ifdef BUILD_CASSANDRA
@@ -51,6 +52,8 @@ struct cassandra_db {
 	CassConsistency read_fallback_consistency, write_fallback_consistency, delete_fallback_consistency;
 	CassLogLevel log_level;
 	unsigned int protocol_version;
+	unsigned int num_threads;
+	unsigned int connect_timeout_secs, request_timeout_secs;
 	in_port_t port;
 
 	CassCluster *cluster;
@@ -366,7 +369,7 @@ driver_cassandra_escape_string(struct sql_db *db ATTR_UNUSED,
 static void driver_cassandra_parse_connect_string(struct cassandra_db *db,
 						  const char *connect_string)
 {
-	const char *const *args, *key, *value;
+	const char *const *args, *key, *value, *error;
 	string_t *hosts = t_str_new(64);
 	bool read_fallback_set = FALSE, write_fallback_set = FALSE, delete_fallback_set = FALSE;
 
@@ -374,6 +377,8 @@ static void driver_cassandra_parse_connect_string(struct cassandra_db *db,
 	db->read_consistency = CASS_CONSISTENCY_LOCAL_QUORUM;
 	db->write_consistency = CASS_CONSISTENCY_LOCAL_QUORUM;
 	db->delete_consistency = CASS_CONSISTENCY_LOCAL_QUORUM;
+	db->connect_timeout_secs = SQL_CONNECT_TIMEOUT_SECS;
+	db->request_timeout_secs = SQL_QUERY_TIMEOUT_SECS;
 
 	args = t_strsplit_spaces(connect_string, " ");
 	for (; *args != NULL; args++) {
@@ -428,6 +433,15 @@ static void driver_cassandra_parse_connect_string(struct cassandra_db *db,
 		} else if (strcmp(key, "version") == 0) {
 			if (str_to_uint(value, &db->protocol_version) < 0)
 				i_fatal("cassandra: Invalid version: %s", value);
+		} else if (strcmp(key, "num_threads") == 0) {
+			if (str_to_uint(value, &db->num_threads) < 0)
+				i_fatal("cassandra: Invalid num_threads: %s", value);
+		} else if (strcmp(key, "connect_timeout") == 0) {
+			if (settings_get_time(value, &db->connect_timeout_secs, &error) < 0)
+				i_fatal("cassandra: Invalid connect_timeout '%s': %s", value, error);
+		} else if (strcmp(key, "request_timeout") == 0) {
+			if (settings_get_time(value, &db->request_timeout_secs, &error) < 0)
+				i_fatal("cassandra: Invalid request_timeout '%s': %s", value, error);
 		} else {
 			i_fatal("cassandra: Unknown connect string: %s", key);
 		}
@@ -463,8 +477,8 @@ static struct sql_db *driver_cassandra_init_v(const char *connect_string)
 	db->timestamp_gen = cass_timestamp_gen_monotonic_new();
 	db->cluster = cass_cluster_new();
 	cass_cluster_set_timestamp_gen(db->cluster, db->timestamp_gen);
-	cass_cluster_set_connect_timeout(db->cluster, SQL_CONNECT_TIMEOUT_SECS * 1000);
-	cass_cluster_set_request_timeout(db->cluster, SQL_QUERY_TIMEOUT_SECS * 1000);
+	cass_cluster_set_connect_timeout(db->cluster, db->connect_timeout_secs * 1000);
+	cass_cluster_set_request_timeout(db->cluster, db->request_timeout_secs * 1000);
 	cass_cluster_set_contact_points(db->cluster, db->hosts);
 	if (db->user != NULL && db->password != NULL)
 		cass_cluster_set_credentials(db->cluster, db->user, db->password);
@@ -472,6 +486,8 @@ static struct sql_db *driver_cassandra_init_v(const char *connect_string)
 		cass_cluster_set_port(db->cluster, db->port);
 	if (db->protocol_version != 0)
 		cass_cluster_set_protocol_version(db->cluster, db->protocol_version);
+	if (db->num_threads != 0)
+		cass_cluster_set_num_threads_io(db->cluster, db->num_threads);
 	db->session = cass_session_new();
 	i_array_init(&db->results, 16);
 	i_array_init(&db->callbacks, 16);

From b7d31e7e72c591364ea7e9cf096ce8ff78db9453 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 11 Apr 2016 17:20:52 +0300
Subject: [PATCH 041/450] fs-posix: Minor code cleanup - removed unnecessary
 code.

If success==TRUE, temp_path==NULL always also.
---
 src/lib-fs/fs-posix.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/lib-fs/fs-posix.c b/src/lib-fs/fs-posix.c
index 3e4a7792375..6f3e99c1c27 100644
--- a/src/lib-fs/fs-posix.c
+++ b/src/lib-fs/fs-posix.c
@@ -47,7 +47,6 @@ struct posix_fs_file {
 	buffer_t *write_buf;
 
 	bool seek_to_beginning;
-	bool success;
 };
 
 struct posix_fs_lock {
@@ -345,7 +344,7 @@ static void fs_posix_file_deinit(struct fs_file *_file)
 	case FS_OPEN_MODE_CREATE_UNIQUE_128:
 	case FS_OPEN_MODE_CREATE:
 	case FS_OPEN_MODE_REPLACE:
-		if (file->success || file->temp_path == NULL)
+		if (file->temp_path == NULL)
 			break;
 		/* failed to create/replace this. delete the temp file */
 		if (unlink(file->temp_path) < 0) {
@@ -470,7 +469,6 @@ static int fs_posix_write_finish(struct posix_fs_file *file)
 		i_unreached();
 	}
 	i_free_and_null(file->temp_path);
-	file->success = TRUE;
 	file->seek_to_beginning = TRUE;
 	/* allow opening the file after writing to it */
 	file->open_mode = FS_OPEN_MODE_READONLY;

From e534a82dc63ad78d9a0aec777193c6a1433289a8 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 11 Apr 2016 17:25:51 +0300
Subject: [PATCH 042/450] fs-posix: If link() fails, preserve its errno through
 the following unlink()

---
 src/lib-fs/fs-posix.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/lib-fs/fs-posix.c b/src/lib-fs/fs-posix.c
index 6f3e99c1c27..ddd9ccb6871 100644
--- a/src/lib-fs/fs-posix.c
+++ b/src/lib-fs/fs-posix.c
@@ -431,7 +431,7 @@ fs_posix_read_stream(struct fs_file *_file, size_t max_buffer_size)
 
 static int fs_posix_write_finish(struct posix_fs_file *file)
 {
-	int ret;
+	int ret, old_errno;
 
 	if ((file->open_flags & FS_OPEN_FLAG_FSYNC) != 0) {
 		if (fdatasync(file->fd) < 0) {
@@ -448,10 +448,12 @@ static int fs_posix_write_finish(struct posix_fs_file *file)
 			fs_set_error(file->file.fs, "link(%s, %s) failed: %m",
 				     file->temp_path, file->full_path);
 		}
+		old_errno = errno;
 		if (unlink(file->temp_path) < 0) {
 			fs_set_error(file->file.fs, "unlink(%s) failed: %m",
 				     file->temp_path);
 		}
+		errno = old_errno;
 		if (ret < 0) {
 			fs_posix_file_close(&file->file);
 			i_free_and_null(file->temp_path);

From c7ea86d2595e597f03dc1e1bae54d93a8578c8ca Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 15 Apr 2016 16:12:44 +0300
Subject: [PATCH 043/450] lib-storage: Don't use same session ID for multiple
 mail_storage_service_next() calls.

dsync does multiple mail_storage_service_next() calls for the same
mail_storage_service_user. This causes stats plugin to send the same
session ID to stats process, which complains about duplicates. Solved this
in a generic way by having the following mail_storage_service_next() calls
append :counter to session ID.
---
 src/lib-storage/mail-storage-service.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/src/lib-storage/mail-storage-service.c b/src/lib-storage/mail-storage-service.c
index babdab9804d..b94d08b5138 100644
--- a/src/lib-storage/mail-storage-service.c
+++ b/src/lib-storage/mail-storage-service.c
@@ -82,6 +82,8 @@ struct mail_storage_service_user {
 	const struct setting_parser_info *user_info;
 	struct setting_parser_context *set_parser;
 
+	unsigned int session_id_counter;
+
 	unsigned int anonymous:1;
 	unsigned int admin:1;
 };
@@ -658,8 +660,15 @@ mail_storage_service_init_post(struct mail_storage_service_ctx *ctx,
 	mail_user->admin = user->admin;
 	mail_user->auth_token = p_strdup(mail_user->pool, user->auth_token);
 	mail_user->auth_user = p_strdup(mail_user->pool, user->auth_user);
-	mail_user->session_id =
-		p_strdup(mail_user->pool, user->input.session_id);
+	if (user->session_id_counter++ == 0) {
+		mail_user->session_id =
+			p_strdup(mail_user->pool, user->input.session_id);
+	} else {
+		mail_user->session_id =
+			p_strdup_printf(mail_user->pool, "%s:%u",
+					user->input.session_id,
+					user->session_id_counter);
+	}
 	mail_user->userdb_fields = user->input.userdb_fields == NULL ? NULL :
 		p_strarray_dup(mail_user->pool, user->input.userdb_fields);
 	mail_user->autoexpunge_enabled =

From 69811d9c7f9306a517a6ce5f5aca2bb9d9e74351 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 25 Apr 2016 14:08:37 +0300
Subject: [PATCH 044/450] dsync: Improved detecting state state string.

---
 src/doveadm/dsync/dsync-mailbox-import.c | 52 ++++++++++++++++--------
 1 file changed, 34 insertions(+), 18 deletions(-)

diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c
index b67febc89b1..1bd127de21e 100644
--- a/src/doveadm/dsync/dsync-mailbox-import.c
+++ b/src/doveadm/dsync/dsync-mailbox-import.c
@@ -145,6 +145,22 @@ imp_debug(struct dsync_mailbox_importer *importer, const char *fmt, ...)
 	} T_END;
 }
 
+static void
+dsync_import_unexpected_state(struct dsync_mailbox_importer *importer,
+			      const char *error)
+{
+	if (!importer->stateful_import) {
+		i_error("Mailbox %s: %s", mailbox_get_vname(importer->box),
+			error);
+	} else {
+		i_warning("Mailbox %s doesn't match previous state: %s "
+			  "(dsync must be run again without the state)",
+			  mailbox_get_vname(importer->box), error);
+	}
+	importer->mail_error = MAIL_ERROR_TEMP;
+	importer->failed = TRUE;
+}
+
 static void
 dsync_mailbox_import_search_init(struct dsync_mailbox_importer *importer)
 {
@@ -267,6 +283,24 @@ dsync_mailbox_import_init(struct mailbox *box,
 	importer->local_initial_highestpvtmodseq = status.highest_pvt_modseq;
 	dsync_mailbox_import_search_init(importer);
 
+	if (!importer->stateful_import)
+		;
+	else if (importer->local_uid_next <= last_common_uid) {
+		dsync_import_unexpected_state(importer, t_strdup_printf(
+			"local UIDNEXT %u <= last common UID %u",
+			importer->local_uid_next, last_common_uid));
+	} else if (importer->local_initial_highestmodseq < last_common_modseq) {
+		dsync_import_unexpected_state(importer, t_strdup_printf(
+			"local HIGHESTMODSEQ %llu < last common HIGHESTMODSEQ %llu",
+			(unsigned long long)importer->local_initial_highestmodseq,
+			(unsigned long long)last_common_modseq));
+	} else if (importer->local_initial_highestpvtmodseq < last_common_pvt_modseq) {
+		dsync_import_unexpected_state(importer, t_strdup_printf(
+			"local HIGHESTMODSEQ %llu < last common HIGHESTMODSEQ %llu",
+			(unsigned long long)importer->local_initial_highestpvtmodseq,
+			(unsigned long long)last_common_pvt_modseq));
+	}
+
 	importer->local_changes = dsync_transaction_log_scan_get_hash(log_scan);
 	importer->local_attr_changes = dsync_transaction_log_scan_get_attr_hash(log_scan);
 	return importer;
@@ -829,20 +863,6 @@ static void dsync_mailbox_save(struct dsync_mailbox_importer *importer,
 	while (!dsync_mailbox_try_save(importer, save_change)) ;
 }
 
-static void
-dsync_import_unexpected_state(struct dsync_mailbox_importer *importer,
-			      const char *error)
-{
-	if (!importer->stateful_import) {
-		i_error("Mailbox %s: %s", mailbox_get_vname(importer->box),
-			error);
-	} else {
-		i_warning("Mailbox %s doesn't match previous state: %s "
-			  "(dsync must be run again without the state)",
-			  mailbox_get_vname(importer->box), error);
-	}
-}
-
 static bool
 dsync_import_set_mail(struct dsync_mailbox_importer *importer,
 		      const struct dsync_mail_change *change)
@@ -871,8 +891,6 @@ dsync_import_set_mail(struct dsync_mailbox_importer *importer,
 		dsync_import_unexpected_state(importer, t_strdup_printf(
 			"Unexpected GUID mismatch for UID=%u: %s != %s",
 			change->uid, guid, cmp_guid));
-		importer->mail_error = MAIL_ERROR_TEMP;
-		importer->failed = TRUE;
 		return FALSE;
 	}
 	return TRUE;
@@ -891,8 +909,6 @@ static bool dsync_check_cur_guid(struct dsync_mailbox_importer *importer,
 		dsync_import_unexpected_state(importer, t_strdup_printf(
 			"Unexpected GUID mismatch (2) for UID=%u: %s != %s",
 			change->uid, importer->cur_guid, cmp_guid));
-		importer->mail_error = MAIL_ERROR_TEMP;
-		importer->failed = TRUE;
 		return FALSE;
 	}
 	return TRUE;

From da0b62ea32c74994171054f9cb551dee49e122d0 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 25 Apr 2016 14:35:21 +0300
Subject: [PATCH 045/450] dsync: If full resync is requested, return empty
 state string.

If the state is wrong, it's better to fully resync all the mailboxes rather
than just the one where a problem was noticed.
---
 src/doveadm/dsync/dsync-brain.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c
index 915d4f23f77..3191708eadc 100644
--- a/src/doveadm/dsync/dsync-brain.c
+++ b/src/doveadm/dsync/dsync-brain.c
@@ -705,6 +705,9 @@ void dsync_brain_get_state(struct dsync_brain *brain, string_t *output)
 	const uint8_t *guid_p;
 	uint8_t *guid;
 
+	if (brain->require_full_resync)
+		return;
+
 	/* update mailbox states */
 	array_foreach(&brain->remote_mailbox_states, new_state) {
 		guid_p = new_state->mailbox_guid;

From 7919e9560109928bd980b4c71ee4a34b51bd4ccd Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 25 Apr 2016 14:38:35 +0300
Subject: [PATCH 046/450] dsync: If state is invalid, exit with code 2 instead
 of tempfail.

We'll have dsync_mailbox_import_*() just return success to everything until
_deinit() is called. The _deinit() will return a failure and set "resync
needed"-flag, which caller will know how to handle.
---
 src/doveadm/dsync/dsync-brain-mailbox.c  |  5 ++++-
 src/doveadm/dsync/dsync-brain-mails.c    | 17 +++++++++++++++--
 src/doveadm/dsync/dsync-mailbox-import.c | 14 +++++++++++---
 src/doveadm/dsync/dsync-mailbox-import.h |  1 +
 4 files changed, 31 insertions(+), 6 deletions(-)

diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c
index f71c7aa808e..fa43537bd97 100644
--- a/src/doveadm/dsync/dsync-brain-mailbox.c
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c
@@ -360,7 +360,7 @@ void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain)
 	if (brain->box_importer != NULL) {
 		uint32_t last_common_uid, last_messages_count;
 		uint64_t last_common_modseq, last_common_pvt_modseq;
-		bool changes_during_sync;
+		bool changes_during_sync, require_full_resync;
 
 		i_assert(brain->failed);
 		(void)dsync_mailbox_import_deinit(&brain->box_importer,
@@ -370,7 +370,10 @@ void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain)
 						  &last_common_pvt_modseq,
 						  &last_messages_count,
 						  &changes_during_sync,
+						  &require_full_resync,
 						  &brain->mail_error);
+		if (require_full_resync)
+			brain->require_full_resync = TRUE;
 	}
 	if (brain->log_scan != NULL)
 		dsync_transaction_log_scan_deinit(&brain->log_scan);
diff --git a/src/doveadm/dsync/dsync-brain-mails.c b/src/doveadm/dsync/dsync-brain-mails.c
index a9ab009c5c9..5a934e4f662 100644
--- a/src/doveadm/dsync/dsync-brain-mails.c
+++ b/src/doveadm/dsync/dsync-brain-mails.c
@@ -219,6 +219,7 @@ static bool dsync_brain_send_mail_request(struct dsync_brain *brain)
 static void dsync_brain_sync_half_finished(struct dsync_brain *brain)
 {
 	struct dsync_mailbox_state state;
+	bool require_full_resync;
 
 	if (brain->box_recv_state < DSYNC_BOX_STATE_RECV_LAST_COMMON ||
 	    brain->box_send_state < DSYNC_BOX_STATE_RECV_LAST_COMMON)
@@ -246,13 +247,25 @@ static void dsync_brain_sync_half_finished(struct dsync_brain *brain)
 						&state.last_common_pvt_modseq,
 						&state.last_messages_count,
 						&state.changes_during_sync,
+						&require_full_resync,
 						&brain->mail_error) < 0) {
-			brain->failed = TRUE;
-			return;
+			if (require_full_resync) {
+				/* don't treat this as brain failure or the
+				   state won't be sent to the other brain.
+				   this also means we'll continue syncing the
+				   following mailboxes. */
+				brain->require_full_resync = TRUE;
+			} else {
+				brain->failed = TRUE;
+			}
 		}
 		if (state.changes_during_sync)
 			brain->changes_during_sync = TRUE;
 	}
+	if (brain->require_full_resync) {
+		state.last_uidvalidity = 0;
+		state.changes_during_sync = TRUE;
+	}
 	brain->mailbox_state = state;
 	dsync_ibc_send_mailbox_state(brain->ibc, &state);
 }
diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c
index 1bd127de21e..f561787dfb7 100644
--- a/src/doveadm/dsync/dsync-mailbox-import.c
+++ b/src/doveadm/dsync/dsync-mailbox-import.c
@@ -108,6 +108,7 @@ struct dsync_mailbox_importer {
 	enum mail_error mail_error;
 
 	unsigned int failed:1;
+	unsigned int require_full_resync:1;
 	unsigned int debug:1;
 	unsigned int stateful_import:1;
 	unsigned int last_common_uid_found:1;
@@ -157,8 +158,7 @@ dsync_import_unexpected_state(struct dsync_mailbox_importer *importer,
 			  "(dsync must be run again without the state)",
 			  mailbox_get_vname(importer->box), error);
 	}
-	importer->mail_error = MAIL_ERROR_TEMP;
-	importer->failed = TRUE;
+	importer->require_full_resync = TRUE;
 }
 
 static void
@@ -1698,6 +1698,8 @@ int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,
 
 	if (importer->failed)
 		return -1;
+	if (importer->require_full_resync)
+		return 0;
 
 	if (!importer->last_common_uid_found) {
 		result = NULL;
@@ -1713,6 +1715,8 @@ int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,
 
 	if (importer->failed)
 		return -1;
+	if (importer->require_full_resync)
+		return 0;
 
 	if (importer->last_common_uid_found) {
 		/* a) uid <= last_common_uid for flag changes and expunges.
@@ -2437,6 +2441,8 @@ int dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer,
 
 	if (importer->failed)
 		return -1;
+	if (importer->require_full_resync)
+		return 0;
 
 	imp_debug(importer, "Import mail body for GUID=%s UID=%u",
 		  mail->guid, mail->uid);
@@ -2750,6 +2756,7 @@ int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **_importer,
 				uint64_t *last_common_pvt_modseq_r,
 				uint32_t *last_messages_count_r,
 				bool *changes_during_sync_r,
+				bool *require_full_resync_r,
 				enum mail_error *error_r)
 {
 	struct dsync_mailbox_importer *importer = *_importer;
@@ -2758,8 +2765,9 @@ int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **_importer,
 
 	*_importer = NULL;
 	*changes_during_sync_r = FALSE;
+	*require_full_resync_r = importer->require_full_resync;
 
-	if (!success && !importer->failed) {
+	if ((!success || importer->require_full_resync) && !importer->failed) {
 		importer->mail_error = MAIL_ERROR_TEMP;
 		importer->failed = TRUE;
 	}
diff --git a/src/doveadm/dsync/dsync-mailbox-import.h b/src/doveadm/dsync/dsync-mailbox-import.h
index d3603ea6215..8139d650ede 100644
--- a/src/doveadm/dsync/dsync-mailbox-import.h
+++ b/src/doveadm/dsync/dsync-mailbox-import.h
@@ -49,6 +49,7 @@ int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **importer,
 				uint64_t *last_common_pvt_modseq_r,
 				uint32_t *last_messages_count_r,
 				bool *changes_during_sync_r,
+				bool *require_full_resync_r,
 				enum mail_error *error_r);
 
 const char *dsync_mailbox_import_get_proctitle(struct dsync_mailbox_importer *importer);

From 30ad986a3ff70921ba116f8a732466a25cc4691d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 25 Apr 2016 16:22:12 +0300
Subject: [PATCH 047/450] lib-mail: Code cleanup to istream-header-filter unit
 test

The exclude_headers arrays were used in a confusing way. The X-Drop header
wasn't actually even used in the exclude_headers.
---
 src/lib-mail/test-istream-header-filter.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/lib-mail/test-istream-header-filter.c b/src/lib-mail/test-istream-header-filter.c
index 6d6629116ac..e59f875c144 100644
--- a/src/lib-mail/test-istream-header-filter.c
+++ b/src/lib-mail/test-istream-header-filter.c
@@ -48,7 +48,7 @@ filter_callback(struct header_filter_istream *input ATTR_UNUSED,
 
 static void test_istream_filter(void)
 {
-	static const char *exclude_headers[] = { "Subject", "To", "X-Drop", NULL };
+	static const char *exclude_headers[] = { "Subject", "To" };
 	const char *input = "From: foo\nFrom: abc\nTo: bar\nSubject: plop\nX-Drop: 1\n\nhello world\n";
 	const char *output = "From: abc\n\nhello world\n";
 	struct istream *istream, *filter, *filter2;
@@ -62,12 +62,14 @@ static void test_istream_filter(void)
 	filter = i_stream_create_header_filter(istream,
 					       HEADER_FILTER_EXCLUDE |
 					       HEADER_FILTER_NO_CR,
-					       exclude_headers, 2,
+					       exclude_headers,
+					       N_ELEMENTS(exclude_headers),
 					       filter_callback, (void *)NULL);
 	filter2 = i_stream_create_header_filter(filter,
 						HEADER_FILTER_EXCLUDE |
 						HEADER_FILTER_NO_CR,
-						exclude_headers, 2,
+						exclude_headers,
+						N_ELEMENTS(exclude_headers),
 						*null_header_filter_callback,
 						(void *)NULL);
 	i_stream_unref(&filter);
@@ -131,7 +133,6 @@ static void test_istream_edit(void)
 
 static void test_istream_end_body_with_lf(void)
 {
-	static const char *empty_strarray[] = { NULL };
 	const char *input = "From: foo\n\nhello world";
 	const char *output = "From: foo\n\nhello world\n";
 	struct istream *istream, *filter;
@@ -147,7 +148,7 @@ static void test_istream_end_body_with_lf(void)
 					       HEADER_FILTER_EXCLUDE |
 					       HEADER_FILTER_NO_CR |
 					       HEADER_FILTER_END_BODY_WITH_LF,
-					       empty_strarray, 0,
+					       NULL, 0,
 					       *null_header_filter_callback,
 					       (void *)NULL);
 

From d7524cf81328d1f16a9651a1b17b5038d53ea46d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 25 Apr 2016 16:25:46 +0300
Subject: [PATCH 048/450] lib-mail: istream-header-filter didn't limit memory
 usage.

A large header was always added to hdr_buf, ignoring any max_buffer_size
limits.
---
 src/lib-mail/istream-header-filter.c      |  8 +++
 src/lib-mail/test-istream-header-filter.c | 88 +++++++++++++++++++++++
 2 files changed, 96 insertions(+)

diff --git a/src/lib-mail/istream-header-filter.c b/src/lib-mail/istream-header-filter.c
index af36a43d8c0..e99a757fa0e 100644
--- a/src/lib-mail/istream-header-filter.c
+++ b/src/lib-mail/istream-header-filter.c
@@ -144,6 +144,9 @@ static ssize_t hdr_stream_update_pos(struct header_filter_istream *mstream)
 	ret = (ssize_t)(pos - mstream->istream.pos - mstream->istream.skip);
 	i_assert(ret >= 0);
 	mstream->istream.pos = pos;
+
+	if (pos >= mstream->istream.max_buffer_size)
+		return -2;
 	return ret;
 }
 
@@ -181,6 +184,9 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 		}
 	}
 
+	if (mstream->hdr_buf->used >= mstream->istream.max_buffer_size)
+		return -2;
+
 	while ((hdr_ret = message_parse_header_next(mstream->hdr_ctx,
 						    &hdr)) > 0) {
 		mstream->cur_line++;
@@ -263,6 +269,8 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 				break;
 			}
 		}
+		if (mstream->hdr_buf->used >= mstream->istream.max_buffer_size)
+			break;
 	}
 
 	if (hdr_ret < 0) {
diff --git a/src/lib-mail/test-istream-header-filter.c b/src/lib-mail/test-istream-header-filter.c
index e59f875c144..400c44ff393 100644
--- a/src/lib-mail/test-istream-header-filter.c
+++ b/src/lib-mail/test-istream-header-filter.c
@@ -98,6 +98,93 @@ static void test_istream_filter(void)
 	test_end();
 }
 
+static void add_random_text(string_t *dest, unsigned int count)
+{
+	unsigned int i;
+
+	for (i = 0; i < count; i++)
+		str_append_c(dest, rand() % ('z'-'a'+1) + 'a');
+}
+
+static void ATTR_NULL(3)
+filter2_callback(struct header_filter_istream *input ATTR_UNUSED,
+		 struct message_header_line *hdr,
+		 bool *matched, void *context ATTR_UNUSED)
+{
+	if (hdr != NULL && strcmp(hdr->name, "To") == 0)
+		*matched = TRUE;
+}
+
+static void test_istream_filter_large_buffer(void)
+{
+	string_t *input, *output;
+	struct istream *istream, *filter;
+	const unsigned char *data;
+	size_t size, prefix_len;
+	const char *p;
+	unsigned int i;
+
+	test_begin("i_stream_create_header_filter(large buffer)");
+
+	input = str_new(default_pool, 1024*128);
+	output = str_new(default_pool, 1024*128);
+	str_append(input, "From: ");
+	add_random_text(input, 1024*31);
+	str_append(input, "\nTo: ");
+	add_random_text(input, 1024*32);
+	str_append(input, "\nSubject: ");
+	add_random_text(input, 1024*34);
+	str_append(input, "\n\nbody\n");
+
+	istream = test_istream_create_data(str_data(input), str_len(input));
+	test_istream_set_max_buffer_size(istream, 8192);
+
+	filter = i_stream_create_header_filter(istream,
+					       HEADER_FILTER_EXCLUDE |
+					       HEADER_FILTER_NO_CR,
+					       NULL, 0,
+					       filter2_callback,
+					       (void *)NULL);
+
+	for (i = 0; i < 2; i++) {
+		for (;;) {
+			ssize_t ret = i_stream_read(filter);
+			i_assert(ret != 0);
+			if (ret == -1)
+				break;
+			if (ret == -2) {
+				data = i_stream_get_data(filter, &size);
+				str_append_n(output, data, size);
+				i_stream_skip(filter, size);
+			}
+		}
+
+		data = i_stream_get_data(filter, &size);
+		test_assert(size <= 8192);
+		str_append_n(output, data, size);
+
+		p = strstr(str_c(input), "To: ");
+		i_assert(p != NULL);
+		prefix_len = p - str_c(input);
+		test_assert(strncmp(str_c(input), str_c(output), prefix_len) == 0);
+
+		p = strchr(p, '\n');
+		i_assert(p != NULL);
+		test_assert(strcmp(p+1, str_c(output) + prefix_len) == 0);
+
+		/* seek back and retry once with caching */
+		i_stream_seek(filter, 0);
+		str_truncate(output, 0);
+	}
+
+	str_free(&input);
+	str_free(&output);
+	i_stream_unref(&filter);
+	i_stream_unref(&istream);
+
+	test_end();
+}
+
 static void ATTR_NULL(3)
 edit_callback(struct header_filter_istream *input,
 	      struct message_header_line *hdr,
@@ -220,6 +307,7 @@ int main(void)
 {
 	static void (*test_functions[])(void) = {
 		test_istream_filter,
+		test_istream_filter_large_buffer,
 		test_istream_edit,
 		test_istream_end_body_with_lf,
 		test_istream_strip_eoh,

From 5fdca8ef228a5dda223aeb3c4b7a51f3e533cb93 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 25 Apr 2016 16:34:59 +0300
Subject: [PATCH 049/450] lib-mail: Improved istream-header-filter unit test.

Existing code already assumes that the callback is called for all the header
data. So it's not used just for filtering headers but also parsing the
header contents. Make this assumption now explicit as a unit test.
---
 src/lib-mail/test-istream-header-filter.c | 53 +++++++++++++++++++++++
 1 file changed, 53 insertions(+)

diff --git a/src/lib-mail/test-istream-header-filter.c b/src/lib-mail/test-istream-header-filter.c
index 400c44ff393..048e3a3a381 100644
--- a/src/lib-mail/test-istream-header-filter.c
+++ b/src/lib-mail/test-istream-header-filter.c
@@ -185,6 +185,58 @@ static void test_istream_filter_large_buffer(void)
 	test_end();
 }
 
+static void
+filter3_callback(struct header_filter_istream *input ATTR_UNUSED,
+		 struct message_header_line *hdr,
+		 bool *matched ATTR_UNUSED, string_t *dest)
+{
+	if (hdr != NULL)
+		message_header_line_write(dest, hdr);
+}
+
+static void test_istream_callbacks(void)
+{
+	string_t *input, *output;
+	struct istream *istream, *filter;
+	unsigned int i;
+
+	test_begin("i_stream_create_header_filter(callbacks)");
+
+	input = str_new(default_pool, 1024*128);
+	output = str_new(default_pool, 1024*128);
+	str_append(input, "From: first line\n ");
+	add_random_text(input, 1024*31);
+	str_append(input, "\nTo: first line\n\tsecond line\n\t");
+	add_random_text(input, 1024*32);
+	str_append(input, "\n last line\nSubject: ");
+	add_random_text(input, 1024*34);
+	str_append(input, "\n");
+
+	istream = test_istream_create_data(str_data(input), str_len(input));
+	test_istream_set_max_buffer_size(istream, 8192);
+
+	filter = i_stream_create_header_filter(istream,
+					       HEADER_FILTER_EXCLUDE |
+					       HEADER_FILTER_NO_CR,
+					       NULL, 0,
+					       filter3_callback,
+					       output);
+
+	/* callback should be called exactly once for all the header input */
+	for (i = 0; i < 2; i++) {
+		while (i_stream_read(filter) != -1)
+			i_stream_skip(filter, i_stream_get_data_size(filter));
+	}
+
+	test_assert(strcmp(str_c(output), str_c(input)) == 0);
+	str_free(&input);
+	str_free(&output);
+	i_stream_unref(&filter);
+	i_stream_unref(&istream);
+
+	test_end();
+}
+
 static void ATTR_NULL(3)
 edit_callback(struct header_filter_istream *input,
 	      struct message_header_line *hdr,
@@ -308,6 +360,7 @@ int main(void)
 	static void (*test_functions[])(void) = {
 		test_istream_filter,
 		test_istream_filter_large_buffer,
+		test_istream_callbacks,
 		test_istream_edit,
 		test_istream_end_body_with_lf,
 		test_istream_strip_eoh,

From ad09af2c9157fbb20c5d2e588f1fac61d87b1b44 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 25 Apr 2016 16:58:30 +0300
Subject: [PATCH 050/450] lib-mail: More fixes to istream-header-filter with
 large input.

Don't assume that when reading the header the second time we'll get exactly
the same header blocks as the first time.

This commit also prevents the filter callback from switching the
matching-decision on non-first header line. This shouldn't be needed and it
could just cause confusion. (It also made it a bit easier to implement this
fix.)
---
 src/lib-mail/istream-header-filter.c      | 37 ++++++++++++++++-------
 src/lib-mail/test-istream-header-filter.c |  4 ++-
 2 files changed, 29 insertions(+), 12 deletions(-)

diff --git a/src/lib-mail/istream-header-filter.c b/src/lib-mail/istream-header-filter.c
index e99a757fa0e..1afb02641ba 100644
--- a/src/lib-mail/istream-header-filter.c
+++ b/src/lib-mail/istream-header-filter.c
@@ -39,6 +39,7 @@ struct header_filter_istream {
 	unsigned int end_body_with_lf:1;
 	unsigned int last_lf_added:1;
 	unsigned int eoh_not_matched:1;
+	unsigned int prev_matched:1;
 };
 
 header_filter_callback *null_header_filter_callback = NULL;
@@ -155,7 +156,6 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 	struct message_header_line *hdr;
 	uoff_t highwater_offset;
 	ssize_t ret, ret2;
-	bool matched;
 	int hdr_ret;
 
 	if (mstream->hdr_ctx == NULL) {
@@ -189,8 +189,10 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 
 	while ((hdr_ret = message_parse_header_next(mstream->hdr_ctx,
 						    &hdr)) > 0) {
-		mstream->cur_line++;
+		bool matched;
 
+		if (!hdr->continued)
+			mstream->cur_line++;
 		if (hdr->eoh) {
 			mstream->seen_eoh = TRUE;
 			matched = TRUE;
@@ -212,15 +214,25 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 			continue;
 		}
 
-		matched = mstream->headers_count == 0 ? FALSE :
-			i_bsearch(hdr->name, mstream->headers,
-				  mstream->headers_count,
-				  sizeof(*mstream->headers),
-				  bsearch_strcasecmp) != NULL;
+		if (hdr->continued) {
+			/* Header line continued - use only the first line's
+			   matched-result. Otherwise multiline headers might
+			   end up being only partially picked, which wouldn't
+			   be very good. However, allow callbacks to modify
+			   the headers in any way they want. */
+			matched = mstream->prev_matched;
+		} else if (mstream->headers_count == 0) {
+			/* no include/exclude headers - default matching */
+			matched = FALSE;
+		} else {
+			matched = i_bsearch(hdr->name, mstream->headers,
+					    mstream->headers_count,
+					    sizeof(*mstream->headers),
+					    bsearch_strcasecmp) != NULL;
+		}
 		if (mstream->callback == NULL) {
 			/* nothing gets excluded */
-		} else if (mstream->cur_line > mstream->parsed_lines ||
-			   mstream->headers_edited) {
+		} else if (!mstream->header_parsed || mstream->headers_edited) {
 			/* first time in this line or we have actually modified
 			   the header so we always want to call the callbacks */
 			bool orig_matched = matched;
@@ -229,18 +241,19 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 			mstream->callback(mstream, hdr, &matched,
 					  mstream->context);
 			if (matched != orig_matched &&
-			    !mstream->headers_edited) {
+			    !hdr->continued && !mstream->headers_edited) {
 				if (!array_is_created(&mstream->match_change_lines))
 					i_array_init(&mstream->match_change_lines, 8);
 				array_append(&mstream->match_change_lines,
 					     &mstream->cur_line, 1);
 			}
-		} else {
+		} else if (!hdr->continued) {
 			/* second time in this line. was it excluded by the
 			   callback the first time? */
 			if (match_line_changed(mstream))
 				matched = !matched;
 		}
+		mstream->prev_matched = matched;
 
 		if (matched == mstream->exclude) {
 			/* ignore */
@@ -303,6 +316,7 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 		mstream->hdr_ctx = NULL;
 
 		if (!mstream->header_parsed && mstream->callback != NULL) {
+			bool matched = FALSE;
 			mstream->callback(mstream, NULL,
 					  &matched, mstream->context);
 			/* check if the callback added more headers.
@@ -427,6 +441,7 @@ i_stream_header_filter_seek_to_header(struct header_filter_istream *mstream,
 		message_parse_header_deinit(&mstream->hdr_ctx);
 	mstream->skip_count = v_offset;
 	mstream->cur_line = 0;
+	mstream->prev_matched = FALSE;
 	mstream->header_read = FALSE;
 	mstream->seen_eoh = FALSE;
 }
diff --git a/src/lib-mail/test-istream-header-filter.c b/src/lib-mail/test-istream-header-filter.c
index 048e3a3a381..cfa544ccb59 100644
--- a/src/lib-mail/test-istream-header-filter.c
+++ b/src/lib-mail/test-istream-header-filter.c
@@ -172,9 +172,11 @@ static void test_istream_filter_large_buffer(void)
 		i_assert(p != NULL);
 		test_assert(strcmp(p+1, str_c(output) + prefix_len) == 0);
 
-		/* seek back and retry once with caching */
+		/* seek back and retry once with caching and different
+		   buffer size */
 		i_stream_seek(filter, 0);
 		str_truncate(output, 0);
+		test_istream_set_max_buffer_size(istream, 4096);
 	}
 
 	str_free(&input);

From 70d210336de9e682a5f4ffd504b269c7be176fda Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 25 Apr 2016 20:13:54 +0300
Subject: [PATCH 051/450] fts-lucene: Cleanup - move code to a separate
 function

---
 src/plugins/fts-lucene/lucene-wrapper.cc | 42 ++++++++++++++----------
 1 file changed, 25 insertions(+), 17 deletions(-)

diff --git a/src/plugins/fts-lucene/lucene-wrapper.cc b/src/plugins/fts-lucene/lucene-wrapper.cc
index 5d634b03b69..6182a0cd9df 100644
--- a/src/plugins/fts-lucene/lucene-wrapper.cc
+++ b/src/plugins/fts-lucene/lucene-wrapper.cc
@@ -819,6 +819,29 @@ rescan_next(struct rescan_context *ctx, Document *doc)
 	}
 }
 
+static void
+rescan_clear_unseen_mailbox(struct rescan_context *rescan_ctx,
+			    const char *vname,
+			    const struct fts_index_header *hdr)
+{
+	struct mailbox *box;
+	struct mailbox_metadata metadata;
+
+	box = mailbox_alloc(rescan_ctx->index->list, vname,
+			    (enum mailbox_flags)0);
+	if (mailbox_open(box) == 0 &&
+	    mailbox_get_metadata(box, MAILBOX_METADATA_GUID,
+				 &metadata) == 0 &&
+	    (rescan_ctx == NULL ||
+	     hash_table_lookup(rescan_ctx->seen_mailbox_guids,
+			       metadata.guid) == NULL)) {
+		/* this mailbox had no records in lucene index.
+		   make sure its last indexed uid is 0 */
+		(void)fts_index_set_header(box, hdr);
+	}
+	mailbox_free(&box);
+}
+
 static void rescan_clear_unseen_mailboxes(struct lucene_index *index,
 					  struct rescan_context *rescan_ctx)
 {
@@ -828,29 +851,14 @@ static void rescan_clear_unseen_mailboxes(struct lucene_index *index,
 		 MAILBOX_LIST_ITER_RETURN_NO_FLAGS);
 	struct mailbox_list_iterate_context *iter;
 	const struct mailbox_info *info;
-	struct mailbox *box;
-	struct mailbox_metadata metadata;
 	struct fts_index_header hdr;
 
 	memset(&hdr, 0, sizeof(hdr));
 	hdr.settings_checksum = fts_lucene_settings_checksum(&index->set);
 
 	iter = mailbox_list_iter_init(index->list, "*", iter_flags);
-	while ((info = mailbox_list_iter_next(iter)) != NULL) {
-		box = mailbox_alloc(index->list, info->vname,
-				    (enum mailbox_flags)0);
-		if (mailbox_open(box) == 0 &&
-		    mailbox_get_metadata(box, MAILBOX_METADATA_GUID,
-					 &metadata) == 0 &&
-		    (rescan_ctx == NULL ||
-		     hash_table_lookup(rescan_ctx->seen_mailbox_guids,
-				       metadata.guid) == NULL)) {
-			/* this mailbox had no records in lucene index.
-			   make sure its last indexed uid is 0 */
-			(void)fts_index_set_header(box, &hdr);
-		}
-		mailbox_free(&box);
-	}
+	while ((info = mailbox_list_iter_next(iter)) != NULL)
+		rescan_clear_unseen_mailbox(rescan_ctx, info->vname, &hdr);
 	(void)mailbox_list_iter_deinit(&iter);
 }
 

From 53d07fbebf837bf3dd621f89a0b46ed528198e43 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 25 Apr 2016 20:14:19 +0300
Subject: [PATCH 052/450] fts-lucene: Rescan skipped namespace roots.

---
 src/plugins/fts-lucene/lucene-wrapper.cc | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/plugins/fts-lucene/lucene-wrapper.cc b/src/plugins/fts-lucene/lucene-wrapper.cc
index 6182a0cd9df..718502290ef 100644
--- a/src/plugins/fts-lucene/lucene-wrapper.cc
+++ b/src/plugins/fts-lucene/lucene-wrapper.cc
@@ -852,6 +852,8 @@ static void rescan_clear_unseen_mailboxes(struct lucene_index *index,
 	struct mailbox_list_iterate_context *iter;
 	const struct mailbox_info *info;
 	struct fts_index_header hdr;
+	struct mail_namespace *ns = index->list->ns;
+	const char *vname;
 
 	memset(&hdr, 0, sizeof(hdr));
 	hdr.settings_checksum = fts_lucene_settings_checksum(&index->set);
@@ -860,6 +862,14 @@ static void rescan_clear_unseen_mailboxes(struct lucene_index *index,
 	while ((info = mailbox_list_iter_next(iter)) != NULL)
 		rescan_clear_unseen_mailbox(rescan_ctx, info->vname, &hdr);
 	(void)mailbox_list_iter_deinit(&iter);
+
+	if (ns->prefix_len > 0 &&
+	    ns->prefix[ns->prefix_len-1] == mail_namespace_get_sep(ns)) {
+		/* namespace prefix itself isn't returned by the listing */
+		vname = t_strndup(index->list->ns->prefix,
+				  index->list->ns->prefix_len-1);
+		rescan_clear_unseen_mailbox(rescan_ctx, vname, &hdr);
+	}
 }
 
 int lucene_index_rescan(struct lucene_index *index)

From 6956285ba13bf2604ab1345df4516f5fee69bd84 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 26 Apr 2016 10:31:49 +0300
Subject: [PATCH 053/450] lib-sql: Fixed building Cassandra as plugin

---
 src/lib-sql/Makefile.am | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/lib-sql/Makefile.am b/src/lib-sql/Makefile.am
index a834a08180f..a6186138d9b 100644
--- a/src/lib-sql/Makefile.am
+++ b/src/lib-sql/Makefile.am
@@ -63,22 +63,22 @@ deplibs = \
 if SQL_PLUGINS
 libdriver_mysql_la_LDFLAGS = -module -avoid-version
 libdriver_mysql_la_LIBADD = $(MYSQL_LIBS)
-libdriver_mysql_la_CPPFLAGS = -I$(top_srcdir)/src/lib $(MYSQL_CFLAGS)
+libdriver_mysql_la_CPPFLAGS = $(AM_CPPFLAGS) $(MYSQL_CFLAGS)
 libdriver_mysql_la_SOURCES = driver-mysql.c
 
 libdriver_pgsql_la_LDFLAGS = -module -avoid-version
 libdriver_pgsql_la_LIBADD = $(PGSQL_LIBS)
-libdriver_pgsql_la_CPPFLAGS = -I$(top_srcdir)/src/lib $(PGSQL_CFLAGS)
+libdriver_pgsql_la_CPPFLAGS = $(AM_CPPFLAGS) $(PGSQL_CFLAGS)
 libdriver_pgsql_la_SOURCES = driver-pgsql.c
 
 libdriver_sqlite_la_LDFLAGS = -module -avoid-version
 libdriver_sqlite_la_LIBADD = $(SQLITE_LIBS)
-libdriver_sqlite_la_CPPFLAGS = -I$(top_srcdir)/src/lib $(SQLITE_CFLAGS)
+libdriver_sqlite_la_CPPFLAGS = $(AM_CPPFLAGS) $(SQLITE_CFLAGS)
 libdriver_sqlite_la_SOURCES = driver-sqlite.c
 
 libdriver_cassandra_la_LDFLAGS = -module -avoid-version
 libdriver_cassandra_la_LIBADD = $(CASSANDRA_LIBS)
-libdriver_cassandra_la_CPPFLAGS = -I$(top_srcdir)/src/lib $(CASSANDRA_CFLAGS)
+libdriver_cassandra_la_CPPFLAGS = $(AM_CPPFLAGS) $(CASSANDRA_CFLAGS)
 libdriver_cassandra_la_SOURCES = driver-cassandra.c
 
 sql_libs =

From c87684e46e9764c4beba561d6ba614fff99670b9 Mon Sep 17 00:00:00 2001
From: Baofeng Wang <baofeng.wang@dovecot.fi>
Date: Tue, 26 Apr 2016 11:10:35 +0300
Subject: [PATCH 054/450] lib-storage: fixup the broken search result with
 \seen or \delete

remove side effects of match_always flag during search.
---
 src/lib-storage/index/index-search.c | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/lib-storage/index/index-search.c b/src/lib-storage/index/index-search.c
index d874da6c782..46d0f198579 100644
--- a/src/lib-storage/index/index-search.c
+++ b/src/lib-storage/index/index-search.c
@@ -985,9 +985,6 @@ static bool search_limit_by_hdr(struct index_search_context *ctx,
 				/* UNSEEN with all seen? */
 				if (args->match_not)
 					return FALSE;
-
-				/* SEEN with all seen */
-				args->match_always = TRUE;
 			} else if (args->match_not) {
 				/* UNSEEN with lowwater limiting */
 				search_limit_lowwater(ctx,
@@ -1005,9 +1002,6 @@ static bool search_limit_by_hdr(struct index_search_context *ctx,
 				/* UNDELETED with all deleted? */
 				if (args->match_not)
 					return FALSE;
-
-				/* DELETED with all deleted */
-				args->match_always = TRUE;
 			} else if (!args->match_not) {
 				/* DELETED with lowwater limiting */
 				search_limit_lowwater(ctx,

From d5804a03e41edd4229075bca57ddac8fefa7ae84 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 26 Apr 2016 11:11:57 +0300
Subject: [PATCH 055/450] lib-storage: Added
 mail_storage_service_all_init_mask()

This allows providing a usermask hint to the userdb iteration lookup. For example
with LDAP this allows setting the base based on the @domain.
---
 src/lib-storage/mail-storage-service.c | 9 ++++++++-
 src/lib-storage/mail-storage-service.h | 5 +++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/lib-storage/mail-storage-service.c b/src/lib-storage/mail-storage-service.c
index b94d08b5138..b347e7e157e 100644
--- a/src/lib-storage/mail-storage-service.c
+++ b/src/lib-storage/mail-storage-service.c
@@ -1529,6 +1529,12 @@ mail_storage_service_all_iter_deinit(struct mail_storage_service_ctx *ctx)
 }
 
 void mail_storage_service_all_init(struct mail_storage_service_ctx *ctx)
+{
+	mail_storage_service_all_init_mask(ctx, "");
+}
+
+void mail_storage_service_all_init_mask(struct mail_storage_service_ctx *ctx,
+					const char *user_mask_hint)
 {
 	enum auth_master_flags flags = 0;
 
@@ -1542,7 +1548,8 @@ void mail_storage_service_all_init(struct mail_storage_service_ctx *ctx)
 		flags |= AUTH_MASTER_FLAG_DEBUG;
 	ctx->iter_conn = auth_master_init(auth_master_get_socket_path(ctx->conn),
 					  flags);
-	ctx->auth_list = auth_master_user_list_init(ctx->iter_conn, "", NULL);
+	ctx->auth_list = auth_master_user_list_init(ctx->iter_conn,
+						    user_mask_hint, NULL);
 }
 
 int mail_storage_service_all_next(struct mail_storage_service_ctx *ctx,
diff --git a/src/lib-storage/mail-storage-service.h b/src/lib-storage/mail-storage-service.h
index 65b633194d1..603aeaccb7f 100644
--- a/src/lib-storage/mail-storage-service.h
+++ b/src/lib-storage/mail-storage-service.h
@@ -111,6 +111,11 @@ int mail_storage_service_lookup_next(struct mail_storage_service_ctx *ctx,
 void mail_storage_service_user_free(struct mail_storage_service_user **user);
 /* Initialize iterating through all users. */
 void mail_storage_service_all_init(struct mail_storage_service_ctx *ctx);
+/* Same as mail_storage_service_all_init(), but give a user mask hint to the
+   userdb iteration lookup. This itself isn't yet guaranteed to filter out any
+   usernames. */
+void mail_storage_service_all_init_mask(struct mail_storage_service_ctx *ctx,
+					const char *user_mask_hint);
 /* Iterate through all usernames. Returns 1 if username was returned, 0 if
    there are no more users, -1 if error. */
 int mail_storage_service_all_next(struct mail_storage_service_ctx *ctx,

From 9de3ff6e97009c87fa0689e41ec15692650baf24 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 26 Apr 2016 11:12:45 +0300
Subject: [PATCH 056/450] doveadm -u: Use mail_storage_service_all_init_mask()

---
 src/doveadm/doveadm-mail.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/doveadm/doveadm-mail.c b/src/doveadm/doveadm-mail.c
index bdb12c13eb9..2b527d6a287 100644
--- a/src/doveadm/doveadm-mail.c
+++ b/src/doveadm/doveadm-mail.c
@@ -450,7 +450,8 @@ doveadm_mail_all_users(struct doveadm_mail_cmd_context *ctx,
 
 	ctx->v.init(ctx, ctx->args);
 
-	mail_storage_service_all_init(ctx->storage_service);
+	mail_storage_service_all_init_mask(ctx->storage_service,
+		wildcard_user != NULL ? wildcard_user : "");
 
 	if (hook_doveadm_mail_init != NULL)
 		hook_doveadm_mail_init(ctx);

From a82c823227ca6d96cfc825687c3d3ba505459e22 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 26 Apr 2016 17:12:25 +0300
Subject: [PATCH 057/450] stats: Support multiple FIFO listeners.

---
 src/stats/fifo-input-connection.c | 16 ++++++++++++++++
 src/stats/fifo-input-connection.h |  2 ++
 src/stats/main.c                  | 16 ++++------------
 3 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/src/stats/fifo-input-connection.c b/src/stats/fifo-input-connection.c
index 64a623d16e9..4c9e060414e 100644
--- a/src/stats/fifo-input-connection.c
+++ b/src/stats/fifo-input-connection.c
@@ -1,6 +1,7 @@
 /* Copyright (c) 2011-2016 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
+#include "llist.h"
 #include "strescape.h"
 #include "istream.h"
 #include "ostream.h"
@@ -15,11 +16,15 @@
 #define MAX_INBUF_SIZE (PIPE_BUF*2)
 
 struct fifo_input_connection {
+	struct fifo_input_connection *prev, *next;
+
 	int fd;
 	struct istream *input;
 	struct io *io;
 };
 
+static struct fifo_input_connection *fifo_conns = NULL;
+
 static int
 fifo_input_connection_request(const char *const *args, const char **error_r)
 {
@@ -75,6 +80,7 @@ struct fifo_input_connection *fifo_input_connection_create(int fd)
 	conn->fd = fd;
 	conn->input = i_stream_create_fd(fd, MAX_INBUF_SIZE, FALSE);
 	conn->io = io_add(fd, IO_READ, fifo_input_connection_input, conn);
+	DLLIST_PREPEND(&fifo_conns, conn);
 	return conn;
 }
 
@@ -84,9 +90,19 @@ void fifo_input_connection_destroy(struct fifo_input_connection **_conn)
 
 	*_conn = NULL;
 
+	DLLIST_REMOVE(&fifo_conns, conn);
 	io_remove(&conn->io);
 	i_stream_destroy(&conn->input);
 	if (close(conn->fd) < 0)
 		i_error("close(conn) failed: %m");
 	i_free(conn);
 }
+
+void fifo_input_connections_destroy_all(void)
+{
+	while (fifo_conns != NULL) {
+		struct fifo_input_connection *conn = fifo_conns;
+
+		fifo_input_connection_destroy(&conn);
+	}
+}
diff --git a/src/stats/fifo-input-connection.h b/src/stats/fifo-input-connection.h
index af6bad644e7..ff394a50603 100644
--- a/src/stats/fifo-input-connection.h
+++ b/src/stats/fifo-input-connection.h
@@ -4,4 +4,6 @@
 struct fifo_input_connection *fifo_input_connection_create(int fd);
 void fifo_input_connection_destroy(struct fifo_input_connection **conn);
 
+void fifo_input_connections_destroy_all(void);
+
 #endif
diff --git a/src/stats/main.c b/src/stats/main.c
index e81ce505f44..c731d548501 100644
--- a/src/stats/main.c
+++ b/src/stats/main.c
@@ -16,20 +16,14 @@
 #include "mail-stats.h"
 #include "client.h"
 
-static struct fifo_input_connection *fifo_input_conn = NULL;
 static struct module *modules = NULL;
 
 static void client_connected(struct master_service_connection *conn)
 {
-	if (conn->fifo) {
-		if (fifo_input_conn != NULL) {
-			i_error("Received another mail-server connection");
-			return;
-		}
-		fifo_input_conn = fifo_input_connection_create(conn->fd);
-	} else {
+	if (conn->fifo)
+		(void)fifo_input_connection_create(conn->fd);
+	else
 		(void)client_create(conn->fd);
-	}
 	master_service_client_connection_accept(conn);
 }
 
@@ -85,6 +79,7 @@ int main(int argc, char *argv[])
 	master_service_run(master_service, client_connected);
 
 	clients_destroy_all();
+	fifo_input_connections_destroy_all();
 	mail_commands_deinit();
 	mail_sessions_deinit();
 	mail_users_deinit();
@@ -92,9 +87,6 @@ int main(int argc, char *argv[])
 	mail_ips_deinit();
 	mail_global_deinit();
 
-	if (fifo_input_conn != NULL)
-		fifo_input_connection_destroy(&fifo_input_conn);
-
 	module_dir_unload(&modules);
 	i_assert(global_used_memory == 0);
 	master_service_deinit(&master_service);

From 4ad29d803f7383a553efcc6f6663dbec89b48bd0 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 26 Apr 2016 21:07:57 +0300
Subject: [PATCH 058/450] fts: Empty "fts" setting should be same as
 nonexistent.

---
 src/plugins/fts/fts-storage.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plugins/fts/fts-storage.c b/src/plugins/fts/fts-storage.c
index 4be64b1e660..e803921ff37 100644
--- a/src/plugins/fts/fts-storage.c
+++ b/src/plugins/fts/fts-storage.c
@@ -805,7 +805,7 @@ void fts_mail_namespaces_added(struct mail_namespace *namespaces)
 	const char *name;
 
 	name = mail_user_plugin_getenv(namespaces->user, "fts");
-	if (name == NULL) {
+	if (name == NULL || name[0] == '\0') {
 		if (namespaces->user->mail_debug)
 			i_debug("fts: No fts setting - plugin disabled");
 		return;

From 9243e07983dcea85817018dcbd6a131240d8fc29 Mon Sep 17 00:00:00 2001
From: Michael Slusarz <michael.slusarz@dovecot.fi>
Date: Tue, 19 Apr 2016 19:24:52 -0600
Subject: [PATCH 059/450] man: Add more complete description of one-way sync

---
 doc/man/doveadm-sync.1.in | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/doc/man/doveadm-sync.1.in b/doc/man/doveadm-sync.1.in
index 2d56391e63c..7b69bbe599c 100644
--- a/doc/man/doveadm-sync.1.in
+++ b/doc/man/doveadm-sync.1.in
@@ -74,6 +74,33 @@ multiple times, then switch mails to be delivered to the new mailbox and
 run
 .B doveadm sync \-1
 once more to transfer any last new mails from the old mailbox.
+.IP
+The one\-way algorithm is the same as two-way dsync algorithm except the
+source account is not modified. It fetches the message's GUID (Global UID),
+which is used to identify any conflicting UIDs in messages. As long as the
+source and destination side has matching UID<\->GUID mapping, those emails
+are assumed to be synced correctly. Only after the first mismatch will
+changes begin.
+.IP
+Example: Source mailbox has messages UID 1..5; source mailbox is sync'd
+using
+.B doveadm backup
+to the destination. Subsequently, UID 6 is delivered to the source mailbox
+and UID 1 is expunged from the destination mailbox. In this example, UID 1
+is kept removed (in destination) because UID 1..5 have identical
+Date+Message\-ID headers. UID 6 is not seen in destination so it's copied.
+.IP
+If both source and destination have UID 6, but the messages are different,
+the headers don't match and both the messages are kept in the destination but
+they're given new UIDs 7 and 8 just to be sure any client didn't get confused
+about what UID 11 actually was. Thus, one\-way sync begins to quickly diverge
+from the source mailbox once changes start to occur on either side; one\-way
+sync should therefore normally only be used within a short period of time
+after a
+.B doveadm backup
+or
+.B doveadm sync
+command was used to synchronize the mailboxes.
 .\"-------------------------------------
 .RE
 .PP

From f184b31abc338e5f01c637bb899aaf217b6f3bda Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 27 Apr 2016 23:47:09 +0300
Subject: [PATCH 060/450] lazy-expunge: Open mailbox with MAILBOX_FLAG_SAVEONLY
 flag.

---
 src/plugins/lazy-expunge/lazy-expunge-plugin.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/plugins/lazy-expunge/lazy-expunge-plugin.c b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
index 8a357d79ce7..89491d71a4a 100644
--- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c
+++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
@@ -109,7 +109,8 @@ mailbox_open_or_create(struct mailbox_list *list, struct mailbox *src_box,
 
 	name = get_dest_vname(list, src_box);
 
-	box = mailbox_alloc(list, name, MAILBOX_FLAG_NO_INDEX_FILES);
+	box = mailbox_alloc(list, name, MAILBOX_FLAG_NO_INDEX_FILES |
+			    MAILBOX_FLAG_SAVEONLY);
 	if (mailbox_open(box) == 0) {
 		*error_r = NULL;
 		return box;

From c05c1b4c1711145ce8bbf9165d31354de32d244b Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 27 Apr 2016 23:58:12 +0300
Subject: [PATCH 061/450] lazy-expunge: Open the mailbox with ACLs ignored.

---
 src/plugins/lazy-expunge/lazy-expunge-plugin.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plugins/lazy-expunge/lazy-expunge-plugin.c b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
index 89491d71a4a..cce392dac53 100644
--- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c
+++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
@@ -110,7 +110,7 @@ mailbox_open_or_create(struct mailbox_list *list, struct mailbox *src_box,
 	name = get_dest_vname(list, src_box);
 
 	box = mailbox_alloc(list, name, MAILBOX_FLAG_NO_INDEX_FILES |
-			    MAILBOX_FLAG_SAVEONLY);
+			    MAILBOX_FLAG_SAVEONLY | MAILBOX_FLAG_IGNORE_ACLS);
 	if (mailbox_open(box) == 0) {
 		*error_r = NULL;
 		return box;

From 65e6f715ba263853d30630808b2cfc43076454bf Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 28 Apr 2016 12:53:49 +0300
Subject: [PATCH 062/450] doveadm dump: Added more index ext records.

"msgs" and "vsize" are now expanded.
---
 src/doveadm/doveadm-dump-index.c | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/src/doveadm/doveadm-dump-index.c b/src/doveadm/doveadm-dump-index.c
index 9edffe754ef..21509cefbaa 100644
--- a/src/doveadm/doveadm-dump-index.c
+++ b/src/doveadm/doveadm-dump-index.c
@@ -77,6 +77,17 @@ struct mailbox_list_index_record {
 	guid_128_t guid;
 	uint32_t uid_validity;
 };
+struct mailbox_list_index_msgs_record {
+	uint32_t messages;
+	uint32_t unseen;
+	uint32_t recent;
+	uint32_t uidnext;
+};
+struct mailbox_index_vsize {
+	uint64_t vsize;
+	uint32_t highest_uid;
+	uint32_t message_count;
+};
 
 struct fts_index_header {
 	uint32_t last_indexed_uid;
@@ -626,6 +637,18 @@ static void dump_record(struct mail_index_view *view, unsigned int seq)
 			printf("                   : parent_uid   = %u\n", lrec->parent_uid);
 			printf("                   : guid         = %s\n", guid_128_to_string(lrec->guid));
 			printf("                   : uid_validity = %u\n", lrec->uid_validity);
+		} else if (strcmp(ext[i].name, "msgs") == 0) {
+			const struct mailbox_list_index_msgs_record *lrec = data;
+			printf("                   : messages = %u\n", lrec->messages);
+			printf("                   : unseen   = %u\n", lrec->unseen);
+			printf("                   : recent   = %u\n", lrec->recent);
+			printf("                   : uidnext  = %u\n", lrec->uidnext);
+		} else if (strcmp(ext[i].name, "vsize") == 0) {
+			const struct mailbox_index_vsize *vrec = data;
+			printf("                   : vsize         = %llu\n",
+			       (unsigned long long)vrec->vsize);
+			printf("                   : highest_uid   = %u\n", vrec->highest_uid);
+			printf("                   : message_count = %u\n", vrec->message_count);
 		}
 	}
 }

From 125840997e07d55eb998eeca72634bd64996586b Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 28 Apr 2016 15:21:41 +0300
Subject: [PATCH 063/450] lib-storage: Added
 index_storage_list_index_has_changed_full()

This allows the caller to use it to implement a slightly different
mailbox.list_index_has_changed()
---
 src/lib-storage/index/index-storage.h | 14 ++++++++++
 src/lib-storage/index/index-sync.c    | 40 +++++++++++++++++++--------
 2 files changed, 42 insertions(+), 12 deletions(-)

diff --git a/src/lib-storage/index/index-storage.h b/src/lib-storage/index/index-storage.h
index 24bf9f2d17e..be43e1d78c3 100644
--- a/src/lib-storage/index/index-storage.h
+++ b/src/lib-storage/index/index-storage.h
@@ -18,6 +18,16 @@ enum mailbox_lock_notify_type {
 	MAILBOX_LOCK_NOTIFY_MAILBOX_OVERRIDE
 };
 
+enum index_storage_list_change {
+	INDEX_STORAGE_LIST_CHANGE_ERROR = -1,
+	INDEX_STORAGE_LIST_CHANGE_NONE = 0,
+	INDEX_STORAGE_LIST_CHANGE_INMEMORY,
+	INDEX_STORAGE_LIST_CHANGE_NORECORD,
+	INDEX_STORAGE_LIST_CHANGE_NOT_IN_FS,
+	INDEX_STORAGE_LIST_CHANGE_SIZE_CHANGED,
+	INDEX_STORAGE_LIST_CHANGE_MTIME_CHANGED
+};
+
 struct index_mailbox_context {
 	union mailbox_module_context module_ctx;
 	enum mail_index_open_flags index_flags;
@@ -155,6 +165,10 @@ bool index_keyword_array_cmp(const ARRAY_TYPE(keyword_indexes) *k1,
 int index_storage_list_index_has_changed(struct mailbox *box,
 					 struct mail_index_view *list_view,
 					 uint32_t seq);
+enum index_storage_list_change
+index_storage_list_index_has_changed_full(struct mailbox *box,
+					  struct mail_index_view *list_view,
+					  uint32_t seq);
 void index_storage_list_index_update_sync(struct mailbox *box,
 					  struct mail_index_transaction *trans,
 					  uint32_t seq);
diff --git a/src/lib-storage/index/index-sync.c b/src/lib-storage/index/index-sync.c
index 79c44329c28..0d4aba0ce4d 100644
--- a/src/lib-storage/index/index-sync.c
+++ b/src/lib-storage/index/index-sync.c
@@ -403,9 +403,10 @@ index_list_get_ext_id(struct mailbox *box, struct mail_index_view *view)
 	return ibox->list_index_sync_ext_id;
 }
 
-int index_storage_list_index_has_changed(struct mailbox *box,
-					 struct mail_index_view *list_view,
-					 uint32_t seq)
+enum index_storage_list_change
+index_storage_list_index_has_changed_full(struct mailbox *box,
+					  struct mail_index_view *list_view,
+					  uint32_t seq)
 {
 	const struct index_storage_list_index_record *rec;
 	const void *data;
@@ -416,7 +417,7 @@ int index_storage_list_index_has_changed(struct mailbox *box,
 	int ret;
 
 	if (mail_index_is_in_memory(mail_index_view_get_index(list_view)))
-		return 1;
+		return INDEX_STORAGE_LIST_CHANGE_INMEMORY;
 
 	ext_id = index_list_get_ext_id(box, list_view);
 	mail_index_lookup_ext(list_view, seq, ext_id, &data, &expunged);
@@ -424,28 +425,43 @@ int index_storage_list_index_has_changed(struct mailbox *box,
 
 	if (rec == NULL || expunged || rec->size == 0 || rec->mtime == 0) {
 		/* doesn't exist / not synced */
-		return 1;
+		return INDEX_STORAGE_LIST_CHANGE_NORECORD;
 	}
 	if (box->storage->set->mailbox_list_index_very_dirty_syncs)
-		return 0;
+		return INDEX_STORAGE_LIST_CHANGE_NONE;
 
 	ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &dir);
 	if (ret < 0)
-		return -1;
+		return INDEX_STORAGE_LIST_CHANGE_ERROR;
 	i_assert(ret > 0);
 
 	path = t_strconcat(dir, "/", box->index_prefix, ".log", NULL);
 	if (stat(path, &st) < 0) {
 		if (errno == ENOENT)
-			return 1;
+			return INDEX_STORAGE_LIST_CHANGE_NOT_IN_FS;
 		mail_storage_set_critical(box->storage,
 					  "stat(%s) failed: %m", path);
-		return -1;
+		return INDEX_STORAGE_LIST_CHANGE_ERROR;
 	}
-	if (rec->size != (st.st_size & 0xffffffffU) ||
-	    rec->mtime != (st.st_mtime & 0xffffffffU))
+	if (rec->size != (st.st_size & 0xffffffffU))
+		return INDEX_STORAGE_LIST_CHANGE_SIZE_CHANGED;
+	if (rec->mtime != (st.st_mtime & 0xffffffffU))
+		return INDEX_STORAGE_LIST_CHANGE_MTIME_CHANGED;
+	return INDEX_STORAGE_LIST_CHANGE_NONE;
+}
+
+int index_storage_list_index_has_changed(struct mailbox *box,
+					 struct mail_index_view *list_view,
+					 uint32_t seq)
+{
+	switch (index_storage_list_index_has_changed_full(box, list_view, seq)) {
+	case INDEX_STORAGE_LIST_CHANGE_ERROR:
+		return -1;
+	case INDEX_STORAGE_LIST_CHANGE_NONE:
+		return 0;
+	default:
 		return 1;
-	return 0;
+	}
 }
 
 void index_storage_list_index_update_sync(struct mailbox *box,

From 18856082d632ac60996637547098688148826b5a Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 02:33:58 +0300
Subject: [PATCH 064/450] lib-storage: mailbox_set_subscribed() ignores
 duplicate changes now.

---
 src/lib-storage/mail-storage.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/lib-storage/mail-storage.c b/src/lib-storage/mail-storage.c
index 454df24a9bc..5c71eb68543 100644
--- a/src/lib-storage/mail-storage.c
+++ b/src/lib-storage/mail-storage.c
@@ -1521,6 +1521,12 @@ int mailbox_set_subscribed(struct mailbox *box, bool set)
 {
 	if (mailbox_verify_name(box) < 0)
 		return -1;
+	if (mailbox_list_iter_subscriptions_refresh(box->list) < 0) {
+		mail_storage_copy_list_error(box->storage, box->list);
+		return -1;
+	}
+	if (mailbox_is_subscribed(box) == set)
+		return 0;
 	return box->v.set_subscribed(box, set);
 }
 

From 46f0240020160618bf3d08df8b75ec63186ebdec Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Thu, 28 Apr 2016 14:03:41 +0300
Subject: [PATCH 065/450] lib-charset: Do not use //IGNORE on non-GNU system

---
 src/lib-charset/charset-iconv.c | 2 ++
 src/lib-charset/test-charset.c  | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/lib-charset/charset-iconv.c b/src/lib-charset/charset-iconv.c
index 964ddec9388..85400a1836c 100644
--- a/src/lib-charset/charset-iconv.c
+++ b/src/lib-charset/charset-iconv.c
@@ -24,6 +24,8 @@ int charset_to_utf8_begin(const char *charset, normalizer_func_t *normalizer,
 	if (charset_is_utf8(charset))
 		cd = (iconv_t)-1;
 	else {
+		if (strcmp(charset, "UTF-8//TEST") == 0)
+			charset = "UTF-8";
 		cd = iconv_open("UTF-8", charset);
 		if (cd == (iconv_t)-1)
 			return -1;
diff --git a/src/lib-charset/test-charset.c b/src/lib-charset/test-charset.c
index 18da108464e..1f763737685 100644
--- a/src/lib-charset/test-charset.c
+++ b/src/lib-charset/test-charset.c
@@ -105,7 +105,7 @@ static void test_charset_iconv(void)
 	}
 	/* Use //IGNORE just to force handling to be done by iconv
 	   instead of our own UTF-8 routines. */
-	test_charset_utf8_common("UTF-8//IGNORE");
+	test_charset_utf8_common("UTF-8//TEST");
 	test_end();
 }
 static void test_charset_iconv_crashes(void)

From 274031f2e9dcab61611ca57e74a283d933285cdd Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 10:53:37 +0300
Subject: [PATCH 066/450] fts-lucene: Fixed crash on rescan

Broken by af2564c7f9e05ad245a032efdfbc5abbb9b70f1e - rescan_ctx can be NULL.
---
 src/plugins/fts-lucene/lucene-wrapper.cc | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/plugins/fts-lucene/lucene-wrapper.cc b/src/plugins/fts-lucene/lucene-wrapper.cc
index 718502290ef..89f99a3fcd9 100644
--- a/src/plugins/fts-lucene/lucene-wrapper.cc
+++ b/src/plugins/fts-lucene/lucene-wrapper.cc
@@ -821,13 +821,14 @@ rescan_next(struct rescan_context *ctx, Document *doc)
 
 static void
 rescan_clear_unseen_mailbox(struct rescan_context *rescan_ctx,
+			    struct mailbox_list *list,
 			    const char *vname,
 			    const struct fts_index_header *hdr)
 {
 	struct mailbox *box;
 	struct mailbox_metadata metadata;
 
-	box = mailbox_alloc(rescan_ctx->index->list, vname,
+	box = mailbox_alloc(list, vname,
 			    (enum mailbox_flags)0);
 	if (mailbox_open(box) == 0 &&
 	    mailbox_get_metadata(box, MAILBOX_METADATA_GUID,
@@ -860,7 +861,7 @@ static void rescan_clear_unseen_mailboxes(struct lucene_index *index,
 
 	iter = mailbox_list_iter_init(index->list, "*", iter_flags);
 	while ((info = mailbox_list_iter_next(iter)) != NULL)
-		rescan_clear_unseen_mailbox(rescan_ctx, info->vname, &hdr);
+		rescan_clear_unseen_mailbox(rescan_ctx, index->list, info->vname, &hdr);
 	(void)mailbox_list_iter_deinit(&iter);
 
 	if (ns->prefix_len > 0 &&
@@ -868,7 +869,7 @@ static void rescan_clear_unseen_mailboxes(struct lucene_index *index,
 		/* namespace prefix itself isn't returned by the listing */
 		vname = t_strndup(index->list->ns->prefix,
 				  index->list->ns->prefix_len-1);
-		rescan_clear_unseen_mailbox(rescan_ctx, vname, &hdr);
+		rescan_clear_unseen_mailbox(rescan_ctx, index->list, vname, &hdr);
 	}
 }
 

From 2dc8e2db12512cb249264276c708a69507e12396 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 12:44:31 +0300
Subject: [PATCH 067/450] configure: Fixed (again) --with-ldap=plugin

---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 95259a2510f..dff5df48d5c 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2044,7 +2044,7 @@ else
   fi
 fi
 AM_CONDITIONAL(LDAP_PLUGIN, test "$have_ldap_plugin" = "yes")
-AM_CONDITIONAL(HAVE_LDAP, test "$want_ldap" = "yes")
+AM_CONDITIONAL(HAVE_LDAP, test "$want_ldap" != "no")
 dict_drivers=
 
 if test $want_db != no; then

From 62219bb5a13b95f1fa4c0d90986f2605b6324738 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 12:34:23 +0300
Subject: [PATCH 068/450] lib-index: Memory allocation optimization

---
 src/lib-index/mail-index-map.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-index/mail-index-map.c b/src/lib-index/mail-index-map.c
index de422cbabf3..52f9ad02abf 100644
--- a/src/lib-index/mail-index-map.c
+++ b/src/lib-index/mail-index-map.c
@@ -327,7 +327,7 @@ static void mail_index_map_copy_records(struct mail_index_record_map *dest,
 	size_t size;
 
 	size = src->records_count * record_size;
-	dest->buffer = buffer_create_dynamic(default_pool, I_MIN(size, 1024));
+	dest->buffer = buffer_create_dynamic(default_pool, I_MAX(size, 1024));
 	buffer_append(dest->buffer, src->records, size);
 
 	dest->records = buffer_get_modifiable_data(dest->buffer, NULL);

From 81bbb8ec793a59f724b7c7d242f2187ff50567c2 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 12:54:46 +0300
Subject: [PATCH 069/450] lib: Implement i_realloc(mem==NULL) more efficiently

Various parts of code use this to allocate the initial buffer. We can
do this more efficiently by using calloc().
---
 src/lib/mempool-system.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/lib/mempool-system.c b/src/lib/mempool-system.c
index ab83e22310c..2cc5d3ae6f6 100644
--- a/src/lib/mempool-system.c
+++ b/src/lib/mempool-system.c
@@ -119,6 +119,10 @@ static void *pool_system_realloc(pool_t pool ATTR_UNUSED, void *mem,
 	if (unlikely(new_size == 0 || new_size > SSIZE_T_MAX))
 		i_panic("Trying to allocate %"PRIuSIZE_T" bytes", new_size);
 
+	if (mem == NULL) {
+		i_assert(old_size == 0);
+		return pool_system_malloc(pool, new_size);
+	}
 #if !defined(USE_GC) && defined(HAVE_MALLOC_USABLE_SIZE)
 	i_assert(old_size == (size_t)-1 || mem == NULL ||
 		 old_size <= malloc_usable_size(mem));

From 83373a25e7d2c6cc674cebea0db94f510110405f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 12:55:52 +0300
Subject: [PATCH 070/450] lib: Use p_malloc() explicitly in first
 buffer_alloc()

Although there is now code in p_realloc() that it should be just as
efficient, this makes profiling somewhat nicer since it can better
distinguish between actual reallocs and initial allocs.
---
 src/lib/buffer.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/lib/buffer.c b/src/lib/buffer.c
index d7430c0fbff..7fb29076f41 100644
--- a/src/lib/buffer.c
+++ b/src/lib/buffer.c
@@ -30,7 +30,10 @@ static void buffer_alloc(struct real_buffer *buf, size_t size)
 
 	i_assert(size > buf->alloc);
 
-	buf->w_buffer = p_realloc(buf->pool, buf->w_buffer, buf->alloc, size);
+	if (buf->w_buffer == NULL)
+		buf->w_buffer = p_malloc(buf->pool, size);
+	else
+		buf->w_buffer = p_realloc(buf->pool, buf->w_buffer, buf->alloc, size);
 	buf->alloc = size;
 
 	buf->r_buffer = buf->w_buffer;

From 9273e1847d0cf0a2945e3c91c9bccb26869686dd Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 13:55:33 +0300
Subject: [PATCH 071/450] lib: buffer_create_dynamic() now adds +1 to init_size

---
 src/lib/buffer.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/lib/buffer.c b/src/lib/buffer.c
index 7fb29076f41..aef163595f4 100644
--- a/src/lib/buffer.c
+++ b/src/lib/buffer.c
@@ -137,7 +137,10 @@ buffer_t *buffer_create_dynamic(pool_t pool, size_t init_size)
 	buf = p_new(pool, struct real_buffer, 1);
 	buf->pool = pool;
 	buf->dynamic = TRUE;
-	buffer_alloc(buf, init_size);
+	/* buffer_alloc() reserves +1 for str_c() NIL, so add +1 here to
+	   init_size so we can actually write that much to the buffer without
+	   realloc */
+	buffer_alloc(buf, init_size+1);
 	return (buffer_t *)buf;
 }
 

From 31a274ee6fd77f9b1814fa18d38b06d6b70c7805 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 14:07:05 +0300
Subject: [PATCH 072/450] lib-index: Use a bit larger initial records buffer
 size

For example with a mailbox having 160k messages the buffer size is around
10MB. Adding just 1% more space to it allows a lot more appends before the
buffer needs to be realloced. This reduces CPU usage quite a lot.
---
 src/lib-index/mail-index-map.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/lib-index/mail-index-map.c b/src/lib-index/mail-index-map.c
index 52f9ad02abf..f3b14f24877 100644
--- a/src/lib-index/mail-index-map.c
+++ b/src/lib-index/mail-index-map.c
@@ -327,7 +327,9 @@ static void mail_index_map_copy_records(struct mail_index_record_map *dest,
 	size_t size;
 
 	size = src->records_count * record_size;
-	dest->buffer = buffer_create_dynamic(default_pool, I_MAX(size, 1024));
+	/* +1% so we have a bit of space to grow. useful for huge mailboxes. */
+	dest->buffer = buffer_create_dynamic(default_pool,
+					     size + I_MAX(size/100, 1024));
 	buffer_append(dest->buffer, src->records, size);
 
 	dest->records = buffer_get_modifiable_data(dest->buffer, NULL);

From ea3a3b6383601ca04a732f09f7c6710158f23a66 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 14:33:45 +0300
Subject: [PATCH 073/450] imapc: Fixed EXPUNGE handling when imapc_features
 didn't have modseq

---
 src/lib-storage/index/imapc/imapc-mailbox.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/lib-storage/index/imapc/imapc-mailbox.c b/src/lib-storage/index/imapc/imapc-mailbox.c
index 349339db429..dce673df3dd 100644
--- a/src/lib-storage/index/imapc/imapc-mailbox.c
+++ b/src/lib-storage/index/imapc/imapc-mailbox.c
@@ -470,7 +470,8 @@ static void imapc_untagged_expunge(const struct imapc_untagged_reply *reply,
 	}
 	uid = imapc_msgmap_rseq_to_uid(msgmap, rseq);
 	imapc_msgmap_expunge(msgmap, rseq);
-	array_delete(&mbox->rseq_modseqs, rseq-1, 1);
+	if (array_is_created(&mbox->rseq_modseqs))
+		array_delete(&mbox->rseq_modseqs, rseq-1, 1);
 
 	imapc_mailbox_init_delayed_trans(mbox);
 	if (mail_index_lookup_seq(mbox->sync_view, uid, &lseq))

From a17f0687f5ad115b93ac962b7ec6d5aec9a92b2b Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Fri, 29 Apr 2016 14:30:46 +0300
Subject: [PATCH 074/450] lib-ldap: Set minimum protocol if supported by
 library

---
 src/lib-ldap/ldap-connection.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/lib-ldap/ldap-connection.c b/src/lib-ldap/ldap-connection.c
index 7b5caa091df..8a468bde44d 100644
--- a/src/lib-ldap/ldap-connection.c
+++ b/src/lib-ldap/ldap-connection.c
@@ -55,10 +55,11 @@ int ldap_connection_setup(struct ldap_connection *conn, const char **error_r)
 	}
 
 	ldap_set_option(conn->conn, LDAP_OPT_X_TLS, &opt);
+#ifdef LDAP_OPT_X_TLS_PROTOCOL_MIN
 	/* refuse to connect to SSLv2 as it's completely insecure */
 	opt = LDAP_OPT_X_TLS_PROTOCOL_SSL3;
 	ldap_set_option(conn->conn, LDAP_OPT_X_TLS_PROTOCOL_MIN, &opt);
-
+#endif
 	opt = conn->set.timeout_secs;
 	/* default timeout */
 	ldap_set_option(conn->conn, LDAP_OPT_TIMEOUT, &opt);

From bd3f28156ad2c5d1c99e0d3e5e9fdfe48694187e Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 15:42:48 +0300
Subject: [PATCH 075/450] quota: Optimize handling a large number of expunges.

This assumes that the mail_expunge() was called in the same order as
sync_notify(), which practically means that they were both done in
ascending uid order. This is usually true.
---
 src/plugins/quota/quota-storage.c | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/plugins/quota/quota-storage.c b/src/plugins/quota/quota-storage.c
index 17d56b4ad9e..52d0e2edaa4 100644
--- a/src/plugins/quota/quota-storage.c
+++ b/src/plugins/quota/quota-storage.c
@@ -31,6 +31,7 @@ struct quota_mailbox {
 	struct quota_transaction_context *expunge_qt;
 	ARRAY(uint32_t) expunge_uids;
 	ARRAY(uoff_t) expunge_sizes;
+	unsigned int prev_idx;
 
 	unsigned int recalculate:1;
 	unsigned int sync_transaction_expunge:1;
@@ -344,10 +345,19 @@ static void quota_mailbox_sync_notify(struct mailbox *box, uint32_t uid,
 		i = count = 0;
 	} else {
 		uids = array_get(&qbox->expunge_uids, &count);
-		for (i = 0; i < count; i++) {
+		for (i = qbox->prev_idx; i < count; i++) {
 			if (uids[i] == uid)
 				break;
 		}
+		if (i >= count) {
+			for (i = 0; i < qbox->prev_idx; i++) {
+				if (uids[i] == uid)
+					break;
+			}
+			if (i == qbox->prev_idx)
+				i = count;
+		}
+		qbox->prev_idx = i;
 	}
 
 	if (qbox->expunge_qt == NULL) {

From 86d26af6f9bd7ba50636b0c6440d7699400289f2 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 15:50:06 +0300
Subject: [PATCH 076/450] global: Use buffer_get_writable_size() where possible

With 09539f3db increasing buffer's init_size with +1 some fts-icu unit tests
started failing. And in general it's better to use the writable size since
that provides the true size that can be used.
---
 src/lib-fts/test-fts-icu.c             | 21 ++++++++++-----------
 src/lib-ssl-iostream/ostream-openssl.c |  2 +-
 src/lib/aqueue.c                       |  4 ++--
 src/lib/str.c                          |  6 +++---
 4 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/src/lib-fts/test-fts-icu.c b/src/lib-fts/test-fts-icu.c
index 6eab62015af..f1748bd63cb 100644
--- a/src/lib-fts/test-fts-icu.c
+++ b/src/lib-fts/test-fts-icu.c
@@ -11,18 +11,17 @@
 
 static void test_fts_icu_utf8_to_utf16_ascii_resize(void)
 {
-	buffer_t *dest = buffer_create_dynamic(pool_datastack_create(), 5);
+	buffer_t *dest = buffer_create_dynamic(pool_datastack_create(), 4);
 
 	test_begin("fts_icu_utf8_to_utf16 ascii resize");
-	/* dynamic buffers reserve +1 for str_c()'s NUL, so 5 -> 4 */
-	test_assert(buffer_get_size(dest) == 5);
+	test_assert(buffer_get_writable_size(dest) == 4);
 	fts_icu_utf8_to_utf16(dest, "12");
 	test_assert(dest->used == 4);
-	test_assert(buffer_get_size(dest) == 5);
+	test_assert(buffer_get_writable_size(dest) == 4);
 
 	fts_icu_utf8_to_utf16(dest, "123");
 	test_assert(dest->used == 6);
-	test_assert(buffer_get_size(dest) == 8);
+	test_assert(buffer_get_writable_size(dest) == 7);
 
 	fts_icu_utf8_to_utf16(dest, "12345");
 	test_assert(dest->used == 10);
@@ -38,7 +37,7 @@ static void test_fts_icu_utf8_to_utf16_32bit_resize(void)
 	test_begin("fts_icu_utf8_to_utf16 32bit resize");
 	for (i = 2; i <= 5; i++) {
 		dest = buffer_create_dynamic(pool_datastack_create(), i);
-		test_assert(buffer_get_size(dest) == i);
+		test_assert(buffer_get_writable_size(dest) == i);
 		fts_icu_utf8_to_utf16(dest, "\xF0\x90\x90\x80"); /* 0x10400 */
 		test_assert(dest->used == 4);
 	}
@@ -69,7 +68,7 @@ static void test_fts_icu_utf16_to_utf8_resize(void)
 	test_begin("fts_icu_utf16_to_utf8 resize");
 	for (i = 2; i <= 6; i++) {
 		dest = t_str_new(i);
-		test_assert(buffer_get_size(dest) == i);
+		test_assert(buffer_get_writable_size(dest) == i);
 		fts_icu_utf16_to_utf8(dest, &src, 1);
 		test_assert(dest->used == 3);
 		test_assert(strcmp(str_c(dest), UNICODE_REPLACEMENT_CHAR_UTF8) == 0);
@@ -131,7 +130,7 @@ static void test_fts_icu_translate_resize(void)
 		buffer_set_used_size(src_utf16, 0);
 		fts_icu_utf8_to_utf16(src_utf16, src_utf8);
 		dest = buffer_create_dynamic(pool_datastack_create(), i);
-		test_assert(buffer_get_size(dest) == i);
+		test_assert(buffer_get_writable_size(dest) == i);
 		test_assert(fts_icu_translate(dest, src_utf16->data,
 					      src_utf16->used/sizeof(UChar),
 					      translit, &error) == 0);
@@ -159,12 +158,12 @@ static void test_fts_icu_lcase_resize(void)
 	unsigned int i;
 
 	test_begin("fts_icu_lcase resize");
-	for (i = 2; i <= 4; i++) {
+	for (i = 1; i <= 3; i++) {
 		dest = t_str_new(i);
-		test_assert(buffer_get_size(dest) == i);
+		test_assert(buffer_get_writable_size(dest) == i);
 		fts_icu_lcase(dest, src);
 		test_assert(strcmp(str_c(dest), "a\xC3\xA4") == 0);
-		test_assert(buffer_get_size(dest) == 4);
+		test_assert(buffer_get_writable_size(dest) == 3);
 	}
 
 	test_end();
diff --git a/src/lib-ssl-iostream/ostream-openssl.c b/src/lib-ssl-iostream/ostream-openssl.c
index 91fc4d9c994..4a8fe9ecb95 100644
--- a/src/lib-ssl-iostream/ostream-openssl.c
+++ b/src/lib-ssl-iostream/ostream-openssl.c
@@ -50,7 +50,7 @@ o_stream_ssl_buffer(struct ssl_ostream *sstream, const struct const_iovec *iov,
 	if (sstream->ostream.max_buffer_size == 0) {
 		/* we're requeted to use whatever space is available in
 		   the buffer */
-		avail = buffer_get_size(sstream->buffer) - sstream->buffer->used;
+		avail = buffer_get_writable_size(sstream->buffer) - sstream->buffer->used;
 	} else {
 		avail = sstream->ostream.max_buffer_size > sstream->buffer->used ?
 			sstream->ostream.max_buffer_size - sstream->buffer->used : 0;
diff --git a/src/lib/aqueue.c b/src/lib/aqueue.c
index 26388bd68ee..efddd10ed95 100644
--- a/src/lib/aqueue.c
+++ b/src/lib/aqueue.c
@@ -10,7 +10,7 @@ struct aqueue *aqueue_init(struct array *array)
 
 	aqueue = i_new(struct aqueue, 1);
 	aqueue->arr = array;
-	aqueue->area_size = buffer_get_size(aqueue->arr->buffer) /
+	aqueue->area_size = buffer_get_writable_size(aqueue->arr->buffer) /
 		aqueue->arr->element_size;
 	i_assert(aqueue->area_size > 0);
 	return aqueue;
@@ -32,7 +32,7 @@ static void aqueue_grow(struct aqueue *aqueue)
 
 	orig_area_size = aqueue->area_size;
 	(void)array_append_space_i(aqueue->arr);
-	aqueue->area_size = buffer_get_size(aqueue->arr->buffer) /
+	aqueue->area_size = buffer_get_writable_size(aqueue->arr->buffer) /
 		aqueue->arr->element_size;
 	i_assert(orig_area_size < aqueue->area_size);
 
diff --git a/src/lib/str.c b/src/lib/str.c
index 4203432aced..8cfd5d207d6 100644
--- a/src/lib/str.c
+++ b/src/lib/str.c
@@ -153,11 +153,11 @@ void str_vprintfa(string_t *str, const char *fmt, va_list args)
 	init_size += SNPRINTF_INITIAL_EXTRA_SIZE;
 
 	/* @UNSAFE */
-	if (pos+init_size > buffer_get_size(str) &&
-	    pos < buffer_get_size(str)) {
+	if (pos+init_size > buffer_get_writable_size(str) &&
+	    pos < buffer_get_writable_size(str)) {
 		/* avoid growing buffer larger if possible. this is also
 		   required if buffer isn't dynamically growing. */
-		init_size = buffer_get_size(str)-pos;
+		init_size = buffer_get_writable_size(str)-pos;
 	}
 	tmp = buffer_get_space_unsafe(str, pos, init_size);
 	ret = vsnprintf(tmp, init_size, fmt, args);

From 316c00c38a5248cdf0ffa0af287beb9f1ade92ae Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 17:20:31 +0300
Subject: [PATCH 077/450] quota-count: Code cleanup - give consistent return
 value

Although the callers didn't care if it returned 0 or 1.
---
 src/plugins/quota/quota-count.c | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/src/plugins/quota/quota-count.c b/src/plugins/quota/quota-count.c
index 15ef14e95cd..a03e17822c3 100644
--- a/src/plugins/quota/quota-count.c
+++ b/src/plugins/quota/quota-count.c
@@ -139,7 +139,7 @@ int quota_count(struct quota_root *root, uint64_t *bytes_r, uint64_t *count_r)
 {
 	struct quota_mailbox_iter *iter;
 	const struct mailbox_info *info;
-	int ret = 0;
+	int ret = 0, ret2;
 
 	*bytes_r = *count_r = 0;
 	if (root->recounting)
@@ -147,13 +147,19 @@ int quota_count(struct quota_root *root, uint64_t *bytes_r, uint64_t *count_r)
 	root->recounting = TRUE;
 
 	iter = quota_mailbox_iter_begin(root);
-	while (ret >= 0 && (info = quota_mailbox_iter_next(iter)) != NULL) {
-		ret = quota_count_mailbox(root, info->ns, info->vname,
-					  bytes_r, count_r);
+	while ((info = quota_mailbox_iter_next(iter)) != NULL) {
+		ret2 = quota_count_mailbox(root, info->ns, info->vname,
+					   bytes_r, count_r);
+		if (ret2 > 0)
+			ret = 1;
+		else if (ret2 < 0) {
+			ret = -1;
+			break;
+		}
 	}
 	quota_mailbox_iter_deinit(&iter);
 	root->recounting = FALSE;
-	return ret;
+	return ret < 0 ? -1 : 0;
 }
 
 static struct quota_root *count_quota_alloc(void)

From b6037984fb0a3409116f21ac241213f543d5d48b Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 17:21:26 +0300
Subject: [PATCH 078/450] quota-count: Cache quota usage within the same ioloop
 run.

This mainly means that when both STORAGE and MESSAGES quota is looked up we
don't need to calculate them twice.
---
 src/plugins/quota/quota-count.c | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/src/plugins/quota/quota-count.c b/src/plugins/quota/quota-count.c
index a03e17822c3..2d2c352ab84 100644
--- a/src/plugins/quota/quota-count.c
+++ b/src/plugins/quota/quota-count.c
@@ -1,9 +1,17 @@
 /* Copyright (c) 2006-2016 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
+#include "ioloop.h"
 #include "mailbox-list-iter.h"
 #include "quota-private.h"
 
+struct count_quota_root {
+	struct quota_root root;
+
+	struct timeval cache_timeval;
+	uint64_t cached_bytes, cached_count;
+};
+
 struct quota_mailbox_iter {
 	struct quota_root *root;
 	struct mail_namespace *ns;
@@ -137,10 +145,19 @@ quota_mailbox_iter_next(struct quota_mailbox_iter *iter)
 
 int quota_count(struct quota_root *root, uint64_t *bytes_r, uint64_t *count_r)
 {
+	struct count_quota_root *croot = (struct count_quota_root *)root;
 	struct quota_mailbox_iter *iter;
 	const struct mailbox_info *info;
 	int ret = 0, ret2;
 
+	if (croot->cache_timeval.tv_usec == ioloop_timeval.tv_usec &&
+	    croot->cache_timeval.tv_sec == ioloop_timeval.tv_sec &&
+	    ioloop_timeval.tv_sec != 0) {
+		*bytes_r = croot->cached_bytes;
+		*count_r = croot->cached_count;
+		return 1;
+	}
+
 	*bytes_r = *count_r = 0;
 	if (root->recounting)
 		return 0;
@@ -159,12 +176,20 @@ int quota_count(struct quota_root *root, uint64_t *bytes_r, uint64_t *count_r)
 	}
 	quota_mailbox_iter_deinit(&iter);
 	root->recounting = FALSE;
+	if (ret > 0) {
+		croot->cache_timeval = ioloop_timeval;
+		croot->cached_bytes = *bytes_r;
+		croot->cached_count = *count_r;
+	}
 	return ret < 0 ? -1 : 0;
 }
 
 static struct quota_root *count_quota_alloc(void)
 {
-	return i_new(struct quota_root, 1);
+	struct count_quota_root *root;
+
+	root = i_new(struct count_quota_root, 1);
+	return &root->root;
 }
 
 static int count_quota_init(struct quota_root *root, const char *args,
@@ -273,6 +298,9 @@ static int
 count_quota_update(struct quota_root *root,
 		   struct quota_transaction_context *ctx)
 {
+	struct count_quota_root *croot = (struct count_quota_root *)root;
+
+	croot->cache_timeval.tv_sec = 0;
 	if (ctx->recalculate) {
 		if (quota_count_recalculate(root) < 0)
 			return -1;

From b3120486f617d6ab2634bb403bef5d1fde761442 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 18:53:04 +0300
Subject: [PATCH 079/450] quota: Fixed quota_over_flag handling when quota
 usage==limit

It couldn't really work correctly otherwise. If we've reached the quota, we
want to start rejecting new mails.
---
 src/plugins/quota/quota.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plugins/quota/quota.c b/src/plugins/quota/quota.c
index d75cee3d288..169278de6f7 100644
--- a/src/plugins/quota/quota.c
+++ b/src/plugins/quota/quota.c
@@ -1029,7 +1029,7 @@ quota_over_flag_check_root(struct mail_user *user, struct quota_root *root)
 				(unsigned long long)value,
 				(unsigned long long)limit);
 		}
-		if (ret > 0 && value > limit)
+		if (ret > 0 && value >= limit)
 			cur_overquota = TRUE;
 	}
 	if (root->quota->set->debug) {

From dcc6ade57671fa0798a0e19766f081932816b539 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 18:55:34 +0300
Subject: [PATCH 080/450] quota: Fixed debug logging for quota_over_flag

---
 src/plugins/quota/quota.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plugins/quota/quota.c b/src/plugins/quota/quota.c
index 169278de6f7..34119b59e0a 100644
--- a/src/plugins/quota/quota.c
+++ b/src/plugins/quota/quota.c
@@ -1034,7 +1034,7 @@ quota_over_flag_check_root(struct mail_user *user, struct quota_root *root)
 	}
 	if (root->quota->set->debug) {
 		i_debug("quota: quota_over_flag=%d(%s) vs currently overquota=%d",
-			overquota_flag, overquota_value != NULL ? "(null)" : overquota_value,
+			overquota_flag, overquota_value == NULL ? "(null)" : overquota_value,
 			cur_overquota);
 	}
 	if (cur_overquota != overquota_flag)

From 1548608e92fa8a1a0ab9ea9622149fb0bc925606 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 18:58:53 +0300
Subject: [PATCH 081/450] quota: Cleanup - split quota_over_flag to init & run
 parts

In preparation for the next change.
---
 src/plugins/quota/quota-private.h |  6 ++++
 src/plugins/quota/quota-storage.c |  2 +-
 src/plugins/quota/quota.c         | 53 +++++++++++++++++--------------
 src/plugins/quota/quota.h         |  2 +-
 4 files changed, 38 insertions(+), 25 deletions(-)

diff --git a/src/plugins/quota/quota-private.h b/src/plugins/quota/quota-private.h
index 5f3e1100ab4..f1e7a2e1b71 100644
--- a/src/plugins/quota/quota-private.h
+++ b/src/plugins/quota/quota-private.h
@@ -131,6 +131,10 @@ struct quota_root {
 	/* Module-specific contexts. See quota_module_id. */
 	ARRAY(void) quota_module_contexts;
 
+	/* Set to the current quota_over_flag, regardless of whether
+	   it matches quota_over_flag_value mask. */
+	const char *quota_over_flag;
+
 	/* don't enforce quota when saving */
 	unsigned int no_enforcing:1;
 	/* If user has unlimited quota, disable quota tracking */
@@ -139,6 +143,8 @@ struct quota_root {
 	unsigned int recounting:1;
 	/* Quota root is hidden (to e.g. IMAP GETQUOTAROOT) */
 	unsigned int hidden:1;
+	/* Is user currently over quota? */
+	unsigned int quota_over_flag_status:1;
 };
 
 struct quota_transaction_context {
diff --git a/src/plugins/quota/quota-storage.c b/src/plugins/quota/quota-storage.c
index 52d0e2edaa4..bc594f05823 100644
--- a/src/plugins/quota/quota-storage.c
+++ b/src/plugins/quota/quota-storage.c
@@ -663,5 +663,5 @@ void quota_mail_namespaces_created(struct mail_namespace *namespaces)
 	for (i = 0; i < count; i++)
 		quota_root_set_namespace(roots[i], namespaces);
 
-	quota_over_flag_check(namespaces->user, quota);
+	quota_over_flag_check(quota);
 }
diff --git a/src/plugins/quota/quota.c b/src/plugins/quota/quota.c
index 34119b59e0a..0b9799c20b6 100644
--- a/src/plugins/quota/quota.c
+++ b/src/plugins/quota/quota.c
@@ -46,6 +46,7 @@ static const struct quota_backend *quota_backends[] = {
 
 static int quota_default_test_alloc(struct quota_transaction_context *ctx,
 				    uoff_t size, bool *too_large_r);
+static void quota_over_flag_check_root(struct quota_root *root);
 
 static const struct quota_backend *quota_backend_find(const char *name)
 {
@@ -983,34 +984,35 @@ int quota_transaction_commit(struct quota_transaction_context **_ctx)
 	return ret;
 }
 
-static void
-quota_over_flag_check_root(struct mail_user *user, struct quota_root *root)
+static void quota_over_flag_init_root(struct quota_root *root)
 {
-	const char *name, *flag_mask, *overquota_value, *overquota_script;
-	const char *const *resources;
-	unsigned int i;
-	uint64_t value, limit;
-	bool overquota_flag, cur_overquota = FALSE;
-	int ret;
-
-	name = t_strconcat(root->set->set_name, "_over_script", NULL);
-	overquota_script = mail_user_plugin_getenv(user, name);
-	if (overquota_script == NULL)
-		return;
+	const char *name, *flag_mask;
 
 	/* e.g.: quota_over_flag_value=TRUE or quota_over_flag_value=*  */
 	name = t_strconcat(root->set->set_name, "_over_flag_value", NULL);
-	flag_mask = mail_user_plugin_getenv(user, name);
+	flag_mask = mail_user_plugin_getenv(root->quota->user, name);
 	if (flag_mask == NULL)
 		return;
 
 	/* compare quota_over_flag's value to quota_over_flag_value and
 	   save the result. */
 	name = t_strconcat(root->set->set_name, "_over_flag", NULL);
-	overquota_value = mail_user_plugin_getenv(user, name);
-	overquota_flag = overquota_value != NULL &&
-		overquota_value[0] != '\0' &&
-		wildcard_match_icase(overquota_value, flag_mask);
+	root->quota_over_flag = p_strdup_empty(root->pool,
+		mail_user_plugin_getenv(root->quota->user, name));
+	root->quota_over_flag_status = root->quota_over_flag != NULL &&
+		wildcard_match_icase(root->quota_over_flag, flag_mask);
+}
+
+static void quota_over_flag_check_root(struct quota_root *root)
+{
+	const char *name, *overquota_script;
+	const char *const *resources;
+	unsigned int i;
+	uint64_t value, limit;
+	bool cur_overquota = FALSE;
+	int ret;
+
+	quota_over_flag_init_root(root);
 
 	resources = quota_root_get_resources(root);
 	for (i = 0; resources[i] != NULL; i++) {
@@ -1034,21 +1036,26 @@ quota_over_flag_check_root(struct mail_user *user, struct quota_root *root)
 	}
 	if (root->quota->set->debug) {
 		i_debug("quota: quota_over_flag=%d(%s) vs currently overquota=%d",
-			overquota_flag, overquota_value == NULL ? "(null)" : overquota_value,
+			root->quota_over_flag_status,
+			root->quota_over_flag == NULL ? "(null)" : root->quota_over_flag,
 			cur_overquota);
 	}
-	if (cur_overquota != overquota_flag)
-		quota_warning_execute(root, overquota_script, overquota_value);
+	if (cur_overquota != root->quota_over_flag_status) {
+		name = t_strconcat(root->set->set_name, "_over_script", NULL);
+		overquota_script = mail_user_plugin_getenv(root->quota->user, name);
+		if (overquota_script != NULL)
+			quota_warning_execute(root, overquota_script, root->quota_over_flag);
+	}
 }
 
-void quota_over_flag_check(struct mail_user *user, struct quota *quota)
+void quota_over_flag_check(struct quota *quota)
 {
 	struct quota_root *const *roots;
 	unsigned int i, count;
 
 	roots = array_get(&quota->roots, &count);
 	for (i = 0; i < count; i++)
-		quota_over_flag_check_root(user, roots[i]);
+		quota_over_flag_check_root(roots[i]);
 }
 
 void quota_transaction_rollback(struct quota_transaction_context **_ctx)
diff --git a/src/plugins/quota/quota.h b/src/plugins/quota/quota.h
index 7b13f7bb788..460d5af7c6c 100644
--- a/src/plugins/quota/quota.h
+++ b/src/plugins/quota/quota.h
@@ -84,6 +84,6 @@ void quota_free_bytes(struct quota_transaction_context *ctx,
 void quota_recalculate(struct quota_transaction_context *ctx);
 
 /* Execute quota_over_scripts if needed. */
-void quota_over_flag_check(struct mail_user *user, struct quota *quota);
+void quota_over_flag_check(struct quota *quota);
 
 #endif

From 31a3bc995702361571fe96dfe0080460b8755f73 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 19:20:11 +0300
Subject: [PATCH 082/450] quota: dict-quota was broken by 464db6d9d

We can't assume that the quota_root given to quota_count() is
count_quota_root, because dict_quota_root also calls it.
---
 src/plugins/quota/quota-count.c | 36 ++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/src/plugins/quota/quota-count.c b/src/plugins/quota/quota-count.c
index 2d2c352ab84..5db0525bf09 100644
--- a/src/plugins/quota/quota-count.c
+++ b/src/plugins/quota/quota-count.c
@@ -145,19 +145,10 @@ quota_mailbox_iter_next(struct quota_mailbox_iter *iter)
 
 int quota_count(struct quota_root *root, uint64_t *bytes_r, uint64_t *count_r)
 {
-	struct count_quota_root *croot = (struct count_quota_root *)root;
 	struct quota_mailbox_iter *iter;
 	const struct mailbox_info *info;
 	int ret = 0, ret2;
 
-	if (croot->cache_timeval.tv_usec == ioloop_timeval.tv_usec &&
-	    croot->cache_timeval.tv_sec == ioloop_timeval.tv_sec &&
-	    ioloop_timeval.tv_sec != 0) {
-		*bytes_r = croot->cached_bytes;
-		*count_r = croot->cached_count;
-		return 1;
-	}
-
 	*bytes_r = *count_r = 0;
 	if (root->recounting)
 		return 0;
@@ -176,10 +167,26 @@ int quota_count(struct quota_root *root, uint64_t *bytes_r, uint64_t *count_r)
 	}
 	quota_mailbox_iter_deinit(&iter);
 	root->recounting = FALSE;
+	return ret;
+}
+
+static int quota_count_cached(struct count_quota_root *root,
+			      uint64_t *bytes_r, uint64_t *count_r)
+{
+	int ret;
+
+	if (root->cache_timeval.tv_usec == ioloop_timeval.tv_usec &&
+	    root->cache_timeval.tv_sec == ioloop_timeval.tv_sec &&
+	    ioloop_timeval.tv_sec != 0) {
+		*bytes_r = root->cached_bytes;
+		*count_r = root->cached_count;
+		return 1;
+	}
+	ret = quota_count(&root->root, bytes_r, count_r);
 	if (ret > 0) {
-		croot->cache_timeval = ioloop_timeval;
-		croot->cached_bytes = *bytes_r;
-		croot->cached_count = *count_r;
+		root->cache_timeval = ioloop_timeval;
+		root->cached_bytes = *bytes_r;
+		root->cached_count = *count_r;
 	}
 	return ret < 0 ? -1 : 0;
 }
@@ -217,12 +224,13 @@ count_quota_root_get_resources(struct quota_root *root ATTR_UNUSED)
 }
 
 static int
-count_quota_get_resource(struct quota_root *root,
+count_quota_get_resource(struct quota_root *_root,
 			 const char *name, uint64_t *value_r)
 {
+	struct count_quota_root *root = (struct count_quota_root *)_root;
 	uint64_t bytes, count;
 
-	if (quota_count(root, &bytes, &count) < 0)
+	if (quota_count_cached(root, &bytes, &count) < 0)
 		return -1;
 
 	if (strcmp(name, QUOTA_NAME_STORAGE_BYTES) == 0)

From ae96eaa83eb68274fea043e8dff2c0cab59e604c Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 19:25:52 +0300
Subject: [PATCH 083/450] quota: Added quota_over_flag_lazy_check flag.

By default the quota_over_flag is checked immediately at startup. With this
option the check is done only at a time when the quota is anyway being read.
---
 src/plugins/quota/quota-private.h |  4 ++++
 src/plugins/quota/quota-storage.c |  2 +-
 src/plugins/quota/quota.c         | 20 +++++++++++++++++---
 src/plugins/quota/quota.h         |  2 +-
 4 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/src/plugins/quota/quota-private.h b/src/plugins/quota/quota-private.h
index f1e7a2e1b71..60683f7d89d 100644
--- a/src/plugins/quota/quota-private.h
+++ b/src/plugins/quota/quota-private.h
@@ -143,8 +143,12 @@ struct quota_root {
 	unsigned int recounting:1;
 	/* Quota root is hidden (to e.g. IMAP GETQUOTAROOT) */
 	unsigned int hidden:1;
+	/* Is quota_over_flag* initialized yet? */
+	unsigned int quota_over_flag_initialized:1;
 	/* Is user currently over quota? */
 	unsigned int quota_over_flag_status:1;
+	/* Did we already check quota_over_flag correctness? */
+	unsigned int quota_over_flag_checked:1;
 };
 
 struct quota_transaction_context {
diff --git a/src/plugins/quota/quota-storage.c b/src/plugins/quota/quota-storage.c
index bc594f05823..34a6c21f47b 100644
--- a/src/plugins/quota/quota-storage.c
+++ b/src/plugins/quota/quota-storage.c
@@ -663,5 +663,5 @@ void quota_mail_namespaces_created(struct mail_namespace *namespaces)
 	for (i = 0; i < count; i++)
 		quota_root_set_namespace(roots[i], namespaces);
 
-	quota_over_flag_check(quota);
+	quota_over_flag_check_startup(quota);
 }
diff --git a/src/plugins/quota/quota.c b/src/plugins/quota/quota.c
index 0b9799c20b6..4e356f0216b 100644
--- a/src/plugins/quota/quota.c
+++ b/src/plugins/quota/quota.c
@@ -656,6 +656,9 @@ const char *quota_root_get_name(struct quota_root *root)
 
 const char *const *quota_root_get_resources(struct quota_root *root)
 {
+	/* if we haven't checked the quota_over_flag yet, do it now */
+	quota_over_flag_check_root(root);
+
 	return root->backend.v.get_resources(root);
 }
 
@@ -988,6 +991,10 @@ static void quota_over_flag_init_root(struct quota_root *root)
 {
 	const char *name, *flag_mask;
 
+	if (root->quota_over_flag_initialized)
+		return;
+	root->quota_over_flag_initialized = TRUE;
+
 	/* e.g.: quota_over_flag_value=TRUE or quota_over_flag_value=*  */
 	name = t_strconcat(root->set->set_name, "_over_flag_value", NULL);
 	flag_mask = mail_user_plugin_getenv(root->quota->user, name);
@@ -1012,6 +1019,9 @@ static void quota_over_flag_check_root(struct quota_root *root)
 	bool cur_overquota = FALSE;
 	int ret;
 
+	if (root->quota_over_flag_checked)
+		return;
+	root->quota_over_flag_checked = TRUE;
 	quota_over_flag_init_root(root);
 
 	resources = quota_root_get_resources(root);
@@ -1048,14 +1058,18 @@ static void quota_over_flag_check_root(struct quota_root *root)
 	}
 }
 
-void quota_over_flag_check(struct quota *quota)
+void quota_over_flag_check_startup(struct quota *quota)
 {
 	struct quota_root *const *roots;
 	unsigned int i, count;
+	const char *name;
 
 	roots = array_get(&quota->roots, &count);
-	for (i = 0; i < count; i++)
-		quota_over_flag_check_root(roots[i]);
+	for (i = 0; i < count; i++) {
+		name = t_strconcat(roots[i]->set->set_name, "_over_flag_lazy_check", NULL);
+		if (mail_user_plugin_getenv(roots[i]->quota->user, name) == NULL)
+			quota_over_flag_check_root(roots[i]);
+	}
 }
 
 void quota_transaction_rollback(struct quota_transaction_context **_ctx)
diff --git a/src/plugins/quota/quota.h b/src/plugins/quota/quota.h
index 460d5af7c6c..04847d1f90d 100644
--- a/src/plugins/quota/quota.h
+++ b/src/plugins/quota/quota.h
@@ -84,6 +84,6 @@ void quota_free_bytes(struct quota_transaction_context *ctx,
 void quota_recalculate(struct quota_transaction_context *ctx);
 
 /* Execute quota_over_scripts if needed. */
-void quota_over_flag_check(struct quota *quota);
+void quota_over_flag_check_startup(struct quota *quota);
 
 #endif

From bbaeb01a32e56b01681843414a8dae28e055f262 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 19:59:36 +0300
Subject: [PATCH 084/450] auth: Session ID wasn't exported to auth workers.

Most importantly this means that log messages didn't contain them.
---
 src/auth/auth-request.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/auth/auth-request.c b/src/auth/auth-request.c
index 90095824b7a..873fd80f393 100644
--- a/src/auth/auth-request.c
+++ b/src/auth/auth-request.c
@@ -265,6 +265,8 @@ void auth_request_export(struct auth_request *request, string_t *dest)
 		str_printfa(dest, "\treal_lport=%u", request->real_local_port);
 	if (request->real_remote_port != 0)
 		str_printfa(dest, "\treal_rport=%u", request->real_remote_port);
+	if (request->session_id != NULL)
+		str_printfa(dest, "\tsession=%s", request->session_id);
 	if (request->debug)
 		str_append(dest, "\tdebug");
 	if (request->secured)
@@ -318,6 +320,7 @@ bool auth_request_import_info(struct auth_request *request,
 		request->debug = TRUE;
 	else
 		return FALSE;
+	/* NOTE: keep in sync with auth_request_export() */
 	return TRUE;
 }
 

From 7f68a693c5a2215984839423089bc849b48d712a Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 20:23:00 +0300
Subject: [PATCH 085/450] auth: Fixed final result in multiple userdbs

We don't want to return the last result's success/failure, but
the entire userdb chain's success/failure.
---
 src/auth/auth-request.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/auth/auth-request.c b/src/auth/auth-request.c
index 873fd80f393..beb707e31ae 100644
--- a/src/auth/auth-request.c
+++ b/src/auth/auth-request.c
@@ -1140,10 +1140,11 @@ void auth_request_userdb_callback(enum userdb_result result,
 		return;
 	}
 
-	if (request->userdb_success)
+	if (request->userdb_success) {
+		result = USERDB_RESULT_OK;
 		userdb_template_export(userdb->override_fields_tmpl, request);
-	else if (request->userdbs_seen_internal_failure ||
-		 result == USERDB_RESULT_INTERNAL_FAILURE) {
+	} else if (request->userdbs_seen_internal_failure ||
+		   result == USERDB_RESULT_INTERNAL_FAILURE) {
 		/* one of the userdb lookups failed. the user might have been
 		   in there, so this is an internal failure */
 		result = USERDB_RESULT_INTERNAL_FAILURE;
@@ -1157,6 +1158,9 @@ void auth_request_userdb_callback(enum userdb_result result,
 			auth_request_log_error(request, AUTH_SUBSYS_MECH,
 				"user not found from any userdbs");
 		}
+		result = USERDB_RESULT_USER_UNKNOWN;
+	} else {
+		result = USERDB_RESULT_USER_UNKNOWN;
 	}
 
 	if (request->userdb_lookup_tempfailed) {

From 48526bfec75c6f199d74b280a9a2b65b1d05eb57 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 29 Apr 2016 21:57:36 +0300
Subject: [PATCH 086/450] lib-ldap: Make static analyzer happier

Remove dead assignment.
---
 src/lib-ldap/ldap-search.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-ldap/ldap-search.c b/src/lib-ldap/ldap-search.c
index 79e3cde785e..132534033e1 100644
--- a/src/lib-ldap/ldap-search.c
+++ b/src/lib-ldap/ldap-search.c
@@ -48,7 +48,7 @@ ldap_search_callback(struct ldap_connection *conn,
 	ret = ldap_parse_result(conn->conn, message, &result_err, NULL,
 				&result_errmsg, NULL, NULL, 0);
 	if (ret == LDAP_NO_RESULTS_RETURNED) {
-		ret = LDAP_SUCCESS;
+		/*ret = LDAP_SUCCESS;*/
 	} else if (ret != LDAP_SUCCESS) {
 		ldap_search_result_failure(req, ret, t_strdup_printf(
 			"ldap_parse_result() failed for search: %s", ldap_err2string(ret)));

From 7f5d5fff77c66971b58e975d839442634c12bc6d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 28 Apr 2016 22:33:14 +0300
Subject: [PATCH 087/450] lib: Set timestamp part more accurately in
 guid_128_generate()

Previously a long-running process would keep the timestamp close to its
original start time. This doesn't really matter as long as GUIDs are treated
opaque, but some pieces of code prefer to try to use the timestamp fields
since they're already there. This makes such code work more nicely.
---
 src/lib/guid.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/lib/guid.c b/src/lib/guid.c
index 902c79658c5..b0524dbb51f 100644
--- a/src/lib/guid.c
+++ b/src/lib/guid.c
@@ -1,6 +1,7 @@
 /* Copyright (c) 2011-2016 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
+#include "ioloop.h"
 #include "buffer.h"
 #include "sha1.h"
 #include "hash.h"
@@ -65,7 +66,15 @@ void guid_128_generate(guid_128_t guid_r)
 		guid_static[2] = (pid & 0x00ff0000) >> 16;
 		guid_static[3] = (pid & 0xff000000) >> 24;
 		guid_128_host_hash_get(my_hostdomain(), guid_static+4);
-	} else if ((uint32_t)ts.tv_nsec < (uint32_t)-1) {
+	} else if (ioloop_timeval.tv_sec > ts.tv_sec ||
+		   (ioloop_timeval.tv_sec == ts.tv_sec &&
+		    ioloop_timeval.tv_usec > ts.tv_nsec*1000)) {
+		/* use ioloop's time since we have it. it doesn't provide any
+		   more uniqueness, but it allows finding out more reliably
+		   when a GUID was created. */
+		ts.tv_sec = ioloop_timeval.tv_sec;
+		ts.tv_nsec = ioloop_timeval.tv_usec*1000;
+	} else if ((uint32_t)ts.tv_nsec < 1000000000) {
 		ts.tv_nsec++;
 	} else {
 		ts.tv_sec++;

From eb83d99d3a21d817ce42cbdf52fbf903d65a405a Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 26 Apr 2016 18:51:55 +0300
Subject: [PATCH 088/450] lmtp: Connect to anvil earlier while still running as
 root

This was a problem only with lmtp_rcpt_check_quota=yes.
---
 src/lmtp/commands.c | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/src/lmtp/commands.c b/src/lmtp/commands.c
index 480e9f2d53b..5d3f9c3e5fb 100644
--- a/src/lmtp/commands.c
+++ b/src/lmtp/commands.c
@@ -179,6 +179,14 @@ parse_xtext(struct client *client, const char *value)
 	return p_strdup(client->state_pool, str_c(str));
 }
 
+static void lmtp_anvil_init(void)
+{
+	if (anvil == NULL) {
+		const char *path = t_strdup_printf("%s/anvil", base_dir);
+		anvil = anvil_client_init(path, NULL, 0);
+	}
+}
+
 int cmd_mail(struct client *client, const char *args)
 {
 	const char *addr, *const *argv;
@@ -212,6 +220,11 @@ int cmd_mail(struct client *client, const char *args)
 	client_send_line(client, "250 2.1.0 OK");
 	client_state_set(client, "MAIL FROM", client->state.mail_from);
 
+	if (client->lmtp_set->lmtp_user_concurrency_limit > 0) {
+		/* connect to anvil before dropping privileges */
+		lmtp_anvil_init();
+	}
+
 	client->state.mail_from_timeval = ioloop_timeval;
 	return 0;
 }
@@ -604,14 +617,6 @@ static void rcpt_anvil_lookup_callback(const char *reply, void *context)
 	}
 }
 
-static void lmtp_anvil_init(void)
-{
-	if (anvil == NULL) {
-		const char *path = t_strdup_printf("%s/anvil", base_dir);
-		anvil = anvil_client_init(path, NULL, 0);
-	}
-}
-
 int cmd_rcpt(struct client *client, const char *args)
 {
 	struct mail_recipient *rcpt;
@@ -720,7 +725,6 @@ int cmd_rcpt(struct client *client, const char *args)
 		const char *query = t_strconcat("LOOKUP\t",
 			master_service_get_name(master_service),
 			"/", str_tabescape(username), NULL);
-		lmtp_anvil_init();
 		client->state.anvil_queries++;
 		rcpt->anvil_query = anvil_client_query(anvil, query,
 					rcpt_anvil_lookup_callback, rcpt);

From 91ccea17ca5d6b7ac798d23357774933ac9524ea Mon Sep 17 00:00:00 2001
From: Michael Slusarz <michael.slusarz@dovecot.fi>
Date: Tue, 26 Apr 2016 14:27:39 -0600
Subject: [PATCH 089/450] man: Add stub for doveadm-fs command

---
 .gitignore              |  1 +
 doc/man/Makefile.am     |  2 +
 doc/man/doveadm-fs.1.in | 93 +++++++++++++++++++++++++++++++++++++++++
 3 files changed, 96 insertions(+)
 create mode 100644 doc/man/doveadm-fs.1.in

diff --git a/.gitignore b/.gitignore
index c4bed011dfb..065f4118d5b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -54,6 +54,7 @@ doc/man/doveadm-exec.1
 doc/man/doveadm-expunge.1
 doc/man/doveadm-fetch.1
 doc/man/doveadm-flags.1
+doc/man/doveadm-fs.1
 doc/man/doveadm-fts.1
 doc/man/doveadm-import.1
 doc/man/doveadm-instance.1
diff --git a/doc/man/Makefile.am b/doc/man/Makefile.am
index 3bfaea3d231..6390ed769ba 100644
--- a/doc/man/Makefile.am
+++ b/doc/man/Makefile.am
@@ -27,6 +27,7 @@ nodist_man1_MANS = \
 	doveadm-expunge.1 \
 	doveadm-fetch.1 \
 	doveadm-flags.1 \
+	doveadm-fs.1 \
 	doveadm-fts.1 \
 	doveadm-import.1 \
 	doveadm-instance.1 \
@@ -75,6 +76,7 @@ EXTRA_DIST = \
 	doveadm-expunge.1.in \
 	doveadm-fetch.1.in \
 	doveadm-flags.1.in \
+	doveadm-fs.1.in \
 	doveadm-fts.1.in \
 	doveadm-import.1.in \
 	doveadm-instance.1.in \
diff --git a/doc/man/doveadm-fs.1.in b/doc/man/doveadm-fs.1.in
new file mode 100644
index 00000000000..0c30f7d2f4b
--- /dev/null
+++ b/doc/man/doveadm-fs.1.in
@@ -0,0 +1,93 @@
+.\" Copyright (c) 2016 Dovecot authors, see the included COPYING file
+.TH DOVEADM\-PROXY 1 "2016-04-26" "Dovecot v2.2" "Dovecot"
+.SH NAME
+doveadm\-fs \- Interact with the abstract mail storage filesystem
+.\"------------------------------------------------------------------------
+.SH SYNOPSIS
+.BR doveadm " [" \-Dv ]
+[\fB\-f\fP \fIformatter\fP]
+.BI fs \ command
+.RI [ ARGUMENTS ]
+.\"------------------------------------------------------------------------
+.SH DESCRIPTION
+The
+.B doveadm fs
+.I commands
+are used to abstractly interact with the storage backend defined in the
+Dovecot configuration. It allows access to the mailbox structure without
+needing to know details of how the storage backend is architected.
+.\"------------------------------------------------------------------------
+@INCLUDE:global-options-formatter@
+.\" --- command specific options --- "/.
+.PP
+This command uses by default the
+.B table
+output formatter.
+.\"------------------------------------------------------------------------
+.SH COMMANDS
+.SS fs copy
+.B doveadm fs copy
+.I fs-driver
+.I fs-args
+.I source-path
+.I dest-path
+.PP
+Copy source path to the destination path.
+.\"-------------------------------------
+.SS fs delete
+.B doveadm fs delete
+[\fB\-R\fP] [\fB\-n\fP \fIcount\fP]
+.I fs-driver
+.I fs-args
+.I path
+[\fIpath\fP ...]
+.PP
+Delete all data associated with the path provided.
+.\"-------------------------------------
+.SS fs get
+.B doveadm fs get
+.I fs-driver
+.I fs-args
+.I path
+.PP
+Retrieve data associated with the path provided.
+.\"-------------------------------------
+.SS fs iter
+.B doveadm fs iter
+.I fs-driver
+.I fs-args
+.I path
+.PP
+Iterate through all data files in the path provided.
+.\"-------------------------------------
+.SS fs iter-dirs
+.B doveadm fs iter-dirs
+.I fs-driver
+.I fs-args
+.I path
+.PP
+Iterate through all directories in the path provided.
+.\"-------------------------------------
+.SS fs put
+.B doveadm fs put
+[\fB\-h\fP \fIhash\fP]
+.I fs-driver
+.I fs-args
+.I input_path
+.I path
+.PP
+Store data at the path provided.
+.\"-------------------------------------
+.SS fs stat
+.B doveadm fs stat
+.I fs-driver
+.I fs-args
+.I path
+.PP
+Retrieve files status for the path provided. Currently, only the total size
+(in bytes) of the item is returned.
+.\"------------------------------------------------------------------------
+@INCLUDE:reporting-bugs@
+.\"------------------------------------------------------------------------
+.SH SEE ALSO
+.BR doveadm (1)

From b58c7f9aabe884d575f7914e6af0874c9a284e11 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sat, 30 Apr 2016 01:05:47 +0300
Subject: [PATCH 090/450] lib-fs: Fixed fs_get_metadata() with fs-metawrap

If mail was already read to EOF.
---
 src/lib-fs/fs-metawrap.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/lib-fs/fs-metawrap.c b/src/lib-fs/fs-metawrap.c
index ee187b96098..0e9f09f70d7 100644
--- a/src/lib-fs/fs-metawrap.c
+++ b/src/lib-fs/fs-metawrap.c
@@ -220,8 +220,12 @@ fs_metawrap_get_metadata(struct fs_file *_file,
 			if (fs_wait_async(_file->fs) < 0)
 				return -1;
 		}
-		if (ret == -1)
+		if (ret == -1 && file->input->stream_errno != 0) {
+			fs_set_error(_file->fs, "read(%s) failed: %s",
+				     i_stream_get_name(file->input),
+				     i_stream_get_error(file->input));
 			return -1;
+		}
 		i_assert(file->metadata_read);
 	}
 	*metadata_r = &_file->metadata;

From 0192511cbe5445f3db5baa65b6206fd6d8d62dcb Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sat, 30 Apr 2016 01:27:27 +0300
Subject: [PATCH 091/450] quota: Fix to earlier commit 62f2f6baf

expunge_uids may become empty with prev_idx > 0
---
 src/plugins/quota/quota-storage.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/plugins/quota/quota-storage.c b/src/plugins/quota/quota-storage.c
index 34a6c21f47b..bd65663b537 100644
--- a/src/plugins/quota/quota-storage.c
+++ b/src/plugins/quota/quota-storage.c
@@ -341,7 +341,8 @@ static void quota_mailbox_sync_notify(struct mailbox *box, uint32_t uid,
 	   try and get the message sizes at this point. Rely on sizes that
 	   we saved earlier, or recalculate the whole quota if we don't know
 	   the size. */
-	if (!array_is_created(&qbox->expunge_uids)) {
+	if (!array_is_created(&qbox->expunge_uids) ||
+	    array_is_empty(&qbox->expunge_uids)) {
 		i = count = 0;
 	} else {
 		uids = array_get(&qbox->expunge_uids, &count);

From 0f6a26edeebd55a24d4ac763ec4088c252181c10 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sat, 30 Apr 2016 14:19:02 +0300
Subject: [PATCH 092/450] lib-mail: message-parser assert-crashfix

Crashes when multipart MIME header is missing end-of-headers line and the
boundary begins with the same prefix as one of the parent boundaries.
Broken by 7a12331c6
---
 src/lib-mail/message-parser.c      |  6 ++-
 src/lib-mail/test-message-parser.c | 76 ++++++++++++++++++++++++++++++
 2 files changed, 81 insertions(+), 1 deletion(-)

diff --git a/src/lib-mail/message-parser.c b/src/lib-mail/message-parser.c
index e3e712a298d..b31b494e757 100644
--- a/src/lib-mail/message-parser.c
+++ b/src/lib-mail/message-parser.c
@@ -173,6 +173,8 @@ message_part_append(pool_t pool, struct message_part *parent)
 	struct message_part *p, *part, **list;
 
 	i_assert(parent != NULL);
+	i_assert((parent->flags & (MESSAGE_PART_FLAG_MULTIPART |
+				   MESSAGE_PART_FLAG_MESSAGE_RFC822)) != 0);
 
 	part = p_new(pool, struct message_part, 1);
 	part->parent = parent;
@@ -313,6 +315,8 @@ static int parse_part_finish(struct message_parser_ctx *ctx,
 	struct message_part *part;
 	size_t line_size;
 
+	i_assert(ctx->last_boundary == NULL);
+
 	/* get back to parent MIME part, summing the child MIME part sizes
 	   into parent's body sizes */
 	for (part = ctx->part; part != boundary->part; part = part->parent) {
@@ -624,8 +628,8 @@ static int parse_next_header(struct message_parser_ctx *ctx,
 		   Content-Type. */
 		i_assert(!ctx->multipart);
 		part->flags = 0;
-		ctx->last_boundary = NULL;
 	}
+	ctx->last_boundary = NULL;
 
 	if (!ctx->part_seen_content_type ||
 	    (part->flags & MESSAGE_PART_FLAG_IS_MIME) == 0) {
diff --git a/src/lib-mail/test-message-parser.c b/src/lib-mail/test-message-parser.c
index 5df4b1387b2..29fce70726a 100644
--- a/src/lib-mail/test-message-parser.c
+++ b/src/lib-mail/test-message-parser.c
@@ -498,6 +498,81 @@ static const char input_msg[] =
 	test_end();
 }
 
+static void test_message_parser_continuing_truncated_mime_boundary(void)
+{
+static const char input_msg[] =
+"Content-Type: multipart/mixed; boundary=\"a\"\n"
+"\n"
+"--a\n"
+"Content-Type: multipart/mixed; boundary=\"ab\"\n"
+"MIME-Version: 1.0\n"
+"--ab\n"
+"Content-Type: text/plain\n"
+"\n"
+"--ab--\n"
+"--a--\n\n";
+	struct message_parser_ctx *parser;
+	struct istream *input;
+	struct message_part *parts, *part;
+	struct message_block block;
+	pool_t pool;
+	int ret;
+
+	test_begin("message parser continuing truncated mime boundary");
+	pool = pool_alloconly_create("message parser", 10240);
+	input = test_istream_create(input_msg);
+
+	parser = message_parser_init(pool, input, 0, 0);
+	while ((ret = message_parser_parse_next_block(parser, &block)) > 0) ;
+	test_assert(ret < 0);
+	message_parser_deinit(&parser, &parts);
+
+	part = parts;
+	test_assert(part->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(part->header_size.lines == 2);
+	test_assert(part->header_size.physical_size == 45);
+	test_assert(part->header_size.virtual_size == 45+2);
+	test_assert(part->body_size.lines == 9);
+	test_assert(part->body_size.physical_size == 112);
+	test_assert(part->body_size.virtual_size == 112+9);
+
+	part = parts->children;
+	test_assert(part->flags == (MESSAGE_PART_FLAG_MULTIPART | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(part->physical_pos == 49);
+	test_assert(part->header_size.lines == 1);
+	test_assert(part->header_size.physical_size == 45+17);
+	test_assert(part->header_size.virtual_size == 45+17+1);
+	test_assert(part->body_size.lines == 0);
+	test_assert(part->body_size.physical_size == 0);
+	test_assert(part->children == NULL);
+
+	/* this will not be a child, since the header was truncated. I guess
+	   we could make it, but it would complicate the message-parser even
+	   more. */
+	part = parts->children->next;
+	test_assert(part->flags == (MESSAGE_PART_FLAG_TEXT | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(part->physical_pos == 117);
+	test_assert(part->header_size.lines == 1);
+	test_assert(part->header_size.physical_size == 25);
+	test_assert(part->header_size.virtual_size == 25+1);
+	test_assert(part->body_size.lines == 0);
+	test_assert(part->body_size.physical_size == 0);
+	test_assert(part->children == NULL);
+
+	part = parts->children->next->next;
+	test_assert(part->flags == (MESSAGE_PART_FLAG_TEXT | MESSAGE_PART_FLAG_IS_MIME));
+	test_assert(part->header_size.lines == 0);
+	test_assert(part->header_size.physical_size == 0);
+	test_assert(part->body_size.lines == 0);
+	test_assert(part->body_size.physical_size == 0);
+	test_assert(part->children == NULL);
+	test_assert(part->next == NULL);
+
+	i_stream_unref(&input);
+	pool_unref(&pool);
+	test_end();
+}
+
 static void test_message_parser_no_eoh(void)
 {
 	static const char input_msg[] = "a:b\n";
@@ -536,6 +611,7 @@ int main(void)
 		test_message_parser_duplicate_mime_boundary,
 		test_message_parser_garbage_suffix_mime_boundary,
 		test_message_parser_continuing_mime_boundary,
+		test_message_parser_continuing_truncated_mime_boundary,
 		test_message_parser_no_eoh,
 		NULL
 	};

From 30c9ffa5fd5d7f6b7d7b8812714eea8c61651135 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sat, 30 Apr 2016 14:55:14 +0300
Subject: [PATCH 093/450] lib-storage: Clear list error before
 mailbox_list_get_hierarchy_sep()

With ACL plugin enabled the call could have triggered dovecot-acl-list
rebuild, which in turn could have set list errors if it didn't have
permissions to all the mailboxes. This caused IMAP logins to fail.
---
 src/lib-storage/mailbox-list.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/lib-storage/mailbox-list.c b/src/lib-storage/mailbox-list.c
index 001667a1e9c..72dc53087a0 100644
--- a/src/lib-storage/mailbox-list.c
+++ b/src/lib-storage/mailbox-list.c
@@ -866,6 +866,10 @@ void mailbox_list_get_default_storage(struct mailbox_list *list,
 
 char mailbox_list_get_hierarchy_sep(struct mailbox_list *list)
 {
+	/* the current API doesn't allow returning an error, so imap code
+	   looks at the list's last error. make sure the error is cleared
+	   so the error-check doesn't return something irrelevant */
+	mailbox_list_clear_error(list);
 	return list->v.get_hierarchy_sep(list);
 }
 

From 9ba5b7f02bbfda58de4b539a85ae35678bfb8419 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 2 May 2016 10:14:20 +0300
Subject: [PATCH 094/450] auth: Added %{auth_user/username/domain} variables

---
 src/auth/auth-request-var-expand.c | 15 ++++++++++++++-
 src/auth/auth-request-var-expand.h |  2 +-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/auth/auth-request-var-expand.c b/src/auth/auth-request-var-expand.c
index 964bbf33b69..f1f041e473b 100644
--- a/src/auth/auth-request-var-expand.c
+++ b/src/auth/auth-request-var-expand.c
@@ -42,6 +42,9 @@ auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT+1] = {
 	{ '\0', NULL, "orig_user" },
 	{ '\0', NULL, "orig_username" },
 	{ '\0', NULL, "orig_domain" },
+	{ '\0', NULL, "auth_user" },
+	{ '\0', NULL, "auth_username" },
+	{ '\0', NULL, "auth_domain" },
 	/* be sure to update AUTH_REQUEST_VAR_TAB_COUNT */
 	{ '\0', NULL, NULL }
 };
@@ -68,7 +71,7 @@ auth_request_get_var_expand_table_full(const struct auth_request *auth_request,
 	const unsigned int auth_count =
 		N_ELEMENTS(auth_request_var_expand_static_tab);
 	struct var_expand_table *tab, *ret_tab;
-	const char *orig_user;
+	const char *orig_user, *auth_user;
 
 	if (escape_func == NULL)
 		escape_func = escape_none;
@@ -154,6 +157,16 @@ auth_request_get_var_expand_table_full(const struct auth_request *auth_request,
 	tab[29].value = strchr(orig_user, '@');
 	if (tab[29].value != NULL)
 		tab[29].value = escape_func(tab[29].value+1, auth_request);
+
+	if (auth_request->master_user != NULL)
+		auth_user = auth_request->master_user;
+	else
+		auth_user = orig_user;
+	tab[30].value = escape_func(auth_user, auth_request);
+	tab[31].value = escape_func(t_strcut(auth_user, '@'), auth_request);
+	tab[32].value = strchr(auth_user, '@');
+	if (tab[32].value != NULL)
+		tab[32].value = escape_func(tab[32].value+1, auth_request);
 	return ret_tab;
 }
 
diff --git a/src/auth/auth-request-var-expand.h b/src/auth/auth-request-var-expand.h
index 180454a4696..1362ec3f861 100644
--- a/src/auth/auth-request-var-expand.h
+++ b/src/auth/auth-request-var-expand.h
@@ -8,7 +8,7 @@ auth_request_escape_func_t(const char *string,
 #define AUTH_REQUEST_VAR_TAB_USER_IDX 0
 #define AUTH_REQUEST_VAR_TAB_USERNAME_IDX 1
 #define AUTH_REQUEST_VAR_TAB_DOMAIN_IDX 2
-#define AUTH_REQUEST_VAR_TAB_COUNT 30
+#define AUTH_REQUEST_VAR_TAB_COUNT 33
 extern const struct var_expand_table
 auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT+1];
 

From 6d1972db0f5f6e503c55fa590d4fbbf2c570d796 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 2 May 2016 12:01:40 +0300
Subject: [PATCH 095/450] imap: Added struct
 client_command_context.tagline_reply

Can be used by plugins.
---
 src/imap/imap-client.c | 1 +
 src/imap/imap-client.h | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/imap/imap-client.c b/src/imap/imap-client.c
index 4144f33bbd4..76424f26b1a 100644
--- a/src/imap/imap-client.c
+++ b/src/imap/imap-client.c
@@ -506,6 +506,7 @@ void client_send_tagline(struct client_command_context *cmd, const char *data)
 
 	i_assert(!cmd->tagline_sent);
 	cmd->tagline_sent = TRUE;
+	cmd->tagline_reply = p_strdup(cmd->pool, data);
 
 	if (tag == NULL || *tag == '\0')
 		tag = "*";
diff --git a/src/imap/imap-client.h b/src/imap/imap-client.h
index 82f2a3987b3..394b59cc19d 100644
--- a/src/imap/imap-client.h
+++ b/src/imap/imap-client.h
@@ -66,6 +66,7 @@ struct client_command_context {
 	   them. */
 	const char *args;
 	enum command_flags cmd_flags;
+	const char *tagline_reply;
 
 	command_func_t *func;
 	void *context;

From f3781e51f41fc3f8404c1bf62f99214d26aeb0f1 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 2 May 2016 14:21:06 +0300
Subject: [PATCH 096/450] dict-ldap: Don't reuse dict for different usernames.

Fixes doing priv/* lookups for multiple usernames.

This currently also means that each username will create a separate LDAP
connection, which isn't ideal. But this is probably better fixed in
lib-ldap code similar to how lib-sql does connection pooling.
---
 src/lib-dict-extra/dict-ldap.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/lib-dict-extra/dict-ldap.c b/src/lib-dict-extra/dict-ldap.c
index 5ea12daa6d1..586f8fc5bc9 100644
--- a/src/lib-dict-extra/dict-ldap.c
+++ b/src/lib-dict-extra/dict-ldap.c
@@ -239,7 +239,8 @@ int ldap_dict_init(struct dict *dict_driver, const char *uri,
 	for(struct ldap_dict *ptr = ldap_dict_list;
 	    ptr != NULL;
 	    ptr = ptr->next) {
-		if (strcmp(ptr->uri, uri) == 0) {
+		if (strcmp(ptr->uri, uri) == 0 &&
+		    null_strcmp(ptr->username, set->username) == 0) {
 			*dict_r = (struct dict*)ptr;
 			return 0;
 		}

From 90afb170719aeed89a8d40a1a01d2bade2a3619b Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 2 May 2016 14:35:40 +0300
Subject: [PATCH 097/450] dict-ldap: Removed caching of ldap-clients

With the previous per-username caching there could be a huge number of
dict-ldaps.
---
 src/lib-dict-extra/dict-ldap.c | 44 ++++++----------------------------
 1 file changed, 7 insertions(+), 37 deletions(-)

diff --git a/src/lib-dict-extra/dict-ldap.c b/src/lib-dict-extra/dict-ldap.c
index 586f8fc5bc9..20dc53328e8 100644
--- a/src/lib-dict-extra/dict-ldap.c
+++ b/src/lib-dict-extra/dict-ldap.c
@@ -46,9 +46,6 @@ struct ldap_dict {
 	struct ldap_dict *prev,*next;
 };
 
-static
-struct ldap_dict *ldap_dict_list;
-
 static
 void ldap_dict_lookup_async(struct dict *dict, const char *key,
 			     dict_lookup_callback_t *callback, void *context);
@@ -200,9 +197,9 @@ const char* ldap_dict_build_query(struct ldap_dict *dict, const struct dict_ldap
 }
 
 static
-int ldap_dict_create(struct dict *dict_driver, const char *uri,
-		     const struct dict_settings *set,
-		     struct dict **dict_r, const char **error_r)
+int ldap_dict_init(struct dict *dict_driver, const char *uri,
+		   const struct dict_settings *set,
+		   struct dict **dict_r, const char **error_r)
 {
 	pool_t pool = pool_alloconly_create("ldap dict", 2048);
 	struct ldap_dict *dict = p_new(pool, struct ldap_dict, 1);
@@ -224,32 +221,16 @@ int ldap_dict_create(struct dict *dict_driver, const char *uri,
 
 	*dict_r = (struct dict*)dict;
 	*error_r = NULL;
-
-	DLLIST_PREPEND(&ldap_dict_list, dict);
-
 	return 0;
 }
 
 static
-int ldap_dict_init(struct dict *dict_driver, const char *uri,
-		   const struct dict_settings *set,
-		   struct dict **dict_r, const char **error_r)
+void ldap_dict_deinit(struct dict *dict)
 {
-	/* reuse possible existing entry */
-	for(struct ldap_dict *ptr = ldap_dict_list;
-	    ptr != NULL;
-	    ptr = ptr->next) {
-		if (strcmp(ptr->uri, uri) == 0 &&
-		    null_strcmp(ptr->username, set->username) == 0) {
-			*dict_r = (struct dict*)ptr;
-			return 0;
-		}
-	}
-	return ldap_dict_create(dict_driver, uri, set, dict_r, error_r);
-}
+	struct ldap_dict *ctx = (struct ldap_dict *)dict;
 
-static
-void ldap_dict_deinit(struct dict *dict ATTR_UNUSED) {
+	ldap_client_deinit(&ctx->client);
+	pool_unref(&ctx->pool);
 }
 
 static
@@ -455,22 +436,11 @@ void dict_ldap_deinit(void);
 void dict_ldap_init(struct module *module ATTR_UNUSED)
 {
 	dict_driver_register(&dict_driver_ldap);
-	ldap_dict_list = NULL;
 }
 
 void dict_ldap_deinit(void)
 {
 	dict_driver_unregister(&dict_driver_ldap);
-	/* destroy all server connections */
-	struct ldap_dict *ptr = ldap_dict_list;
-	ldap_dict_list = NULL;
-
-	while(ptr != NULL) {
-		ldap_client_deinit(&(ptr->client));
-		pool_t pool = ptr->pool;
-		ptr = ptr->next;
-		pool_unref(&pool);
-	}
 }
 
 const char *dict_ldap_plugin_dependencies[] = { NULL };

From 527dad09b4f11062bf34ce7ea5150a4e94c5185a Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 2 May 2016 15:20:18 +0300
Subject: [PATCH 098/450] lib-ldap: Added initial connection pooling code.

This is mainly about allowing multiple dict-ldaps to use the same
ldap-connection. In future we could support load balancing with multiple
concurrent LDAP connections.
---
 src/lib-dict-extra/dict-ldap.c      |   1 +
 src/lib-ldap/Makefile.am            |   2 +
 src/lib-ldap/ldap-client.c          |  35 +++++++--
 src/lib-ldap/ldap-client.h          |   7 ++
 src/lib-ldap/ldap-connection-pool.c | 113 ++++++++++++++++++++++++++++
 src/lib-ldap/ldap-connection-pool.h |  27 +++++++
 src/lib-ldap/ldap-connection.c      |  38 ++++++++++
 src/lib-ldap/ldap-private.h         |   2 +
 8 files changed, 217 insertions(+), 8 deletions(-)
 create mode 100644 src/lib-ldap/ldap-connection-pool.c
 create mode 100644 src/lib-ldap/ldap-connection-pool.h

diff --git a/src/lib-dict-extra/dict-ldap.c b/src/lib-dict-extra/dict-ldap.c
index 20dc53328e8..b9d19dd5d07 100644
--- a/src/lib-dict-extra/dict-ldap.c
+++ b/src/lib-dict-extra/dict-ldap.c
@@ -440,6 +440,7 @@ void dict_ldap_init(struct module *module ATTR_UNUSED)
 
 void dict_ldap_deinit(void)
 {
+	ldap_clients_cleanup();
 	dict_driver_unregister(&dict_driver_ldap);
 }
 
diff --git a/src/lib-ldap/Makefile.am b/src/lib-ldap/Makefile.am
index fa8e213fb34..b990e93fdf3 100644
--- a/src/lib-ldap/Makefile.am
+++ b/src/lib-ldap/Makefile.am
@@ -11,6 +11,7 @@ AM_CPPFLAGS = \
 libdovecot_ldap_la_SOURCES = \
 	ldap-client.c \
 	ldap-connection.c \
+	ldap-connection-pool.c \
 	ldap-iterator.c \
 	ldap-search.c \
 	ldap-compare.c \
@@ -24,6 +25,7 @@ headers = \
 	ldap-client.h
 
 noinst_HEADERS = \
+	ldap-connection-pool.h \
 	ldap-private.h
 
 pkginc_libdir=$(pkgincludedir)
diff --git a/src/lib-ldap/ldap-client.c b/src/lib-ldap/ldap-client.c
index d8b9743dbea..bdafd87b115 100644
--- a/src/lib-ldap/ldap-client.c
+++ b/src/lib-ldap/ldap-client.c
@@ -1,21 +1,31 @@
 /* Copyright (c) 2016 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
+#include "ldap-connection-pool.h"
 #include "ldap-private.h"
 
+/* Max number of ldap-connections that can be created. For now this is
+   unlimited since we're assuming our callers aren't calling us with many
+   different settings. */
+#define LDAP_CONN_POOL_MAX_CONNECTIONS UINT_MAX
+
 struct ldap_client {
-	/* for now we support just a single connection, but this could be
-	   extended to a connection pool. */
-	struct ldap_connection *conn;
+	struct ldap_connection_list *list;
 };
 
+static struct ldap_connection_pool *ldap_conn_pool = NULL;
+
 int ldap_client_init(const struct ldap_client_settings *set,
 		     struct ldap_client **client_r, const char **error_r)
 {
 	struct ldap_client *client;
 
+	if (ldap_conn_pool == NULL)
+		ldap_conn_pool = ldap_connection_pool_init(LDAP_CONN_POOL_MAX_CONNECTIONS);
+
 	client = i_new(struct ldap_client, 1);
-	if (ldap_connection_init(client, set, &client->conn, error_r) < 0) {
+	if (ldap_connection_pool_get(ldap_conn_pool, client, set,
+				     &client->list, error_r) < 0) {
 		i_free(client);
 		return -1;
 	}
@@ -29,13 +39,13 @@ void ldap_client_deinit(struct ldap_client **_client)
 
 	*_client = NULL;
 
-	ldap_connection_deinit(&client->conn);
+	ldap_connection_pool_unref(ldap_conn_pool, &client->list);
 	i_free(client);
 }
 
 void ldap_client_switch_ioloop(struct ldap_client *client)
 {
-	ldap_connection_switch_ioloop(client->conn);
+	ldap_connection_switch_ioloop(client->list->conn);
 }
 
 #undef ldap_search_start
@@ -43,7 +53,9 @@ void ldap_search_start(struct ldap_client *client,
 		       const struct ldap_search_input *input,
 		       ldap_result_callback_t *callback, void *context)
 {
-	return ldap_connection_search_start(client->conn, input, callback, context);
+	/* FIXME: we could support multiple concurrent LDAP connections to
+	   the same host. */
+	return ldap_connection_search_start(client->list->conn, input, callback, context);
 }
 
 #undef ldap_compare_start
@@ -51,5 +63,12 @@ void ldap_compare_start(struct ldap_client *client,
 			const struct ldap_compare_input *input,
 			ldap_result_callback_t *callback, void *context)
 {
-	return ldap_connection_compare_start(client->conn, input, callback, context);
+	return ldap_connection_compare_start(client->list->conn, input, callback, context);
+}
+
+void ldap_clients_cleanup(void)
+{
+	if (ldap_conn_pool != NULL &&
+	    !ldap_connection_pool_have_references(ldap_conn_pool))
+		ldap_connection_pool_deinit(&ldap_conn_pool);
 }
diff --git a/src/lib-ldap/ldap-client.h b/src/lib-ldap/ldap-client.h
index 5ce14dc6bf7..f0707c87fbe 100644
--- a/src/lib-ldap/ldap-client.h
+++ b/src/lib-ldap/ldap-client.h
@@ -19,6 +19,8 @@ struct ldap_entry;
 typedef void ldap_result_callback_t(struct ldap_result *result, void *context);
 
 struct ldap_client_settings {
+	/* NOTE: when adding here, remember to update
+	   ldap_connection_have_settings() and ldap_connection_init() */
 	const char *uri;
 	const char *bind_dn;
 	const char *password;
@@ -58,6 +60,11 @@ int ldap_client_init(const struct ldap_client_settings *set,
 void ldap_client_deinit(struct ldap_client **client);
 void ldap_client_switch_ioloop(struct ldap_client *client);
 
+/* Deinitialize all pooled LDAP connections if there are no references left.
+   This allows freeing the memory at deinit, but still allows multiple
+   independent code parts to use lib-ldap and call this function. */
+void ldap_clients_cleanup(void);
+
 void ldap_search_start(struct ldap_client *client,
 		       const struct ldap_search_input *input,
 		       ldap_result_callback_t *callback,
diff --git a/src/lib-ldap/ldap-connection-pool.c b/src/lib-ldap/ldap-connection-pool.c
new file mode 100644
index 00000000000..8b2d7a997f3
--- /dev/null
+++ b/src/lib-ldap/ldap-connection-pool.c
@@ -0,0 +1,113 @@
+/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "llist.h"
+#include "ldap-private.h"
+#include "ldap-connection-pool.h"
+
+struct ldap_connection_pool {
+	struct ldap_connection_list *conn_list;
+	unsigned int conn_count;
+
+	unsigned int max_connections;
+};
+
+static void ldap_connection_list_remove(struct ldap_connection_pool *pool,
+					struct ldap_connection_list *list)
+{
+	DLLIST_REMOVE(&pool->conn_list, list);
+	pool->conn_count--;
+
+	ldap_connection_deinit(&list->conn);
+	i_free(list);
+}
+
+static void
+ldap_connection_pool_shrink_to(struct ldap_connection_pool *pool,
+			       unsigned int max_count)
+{
+	struct ldap_connection_list *list, *next;
+
+	list = pool->conn_list;
+	for (; list != NULL && pool->conn_count > max_count; list = next) {
+		next = list->next;
+		if (list->refcount == 0)
+			ldap_connection_list_remove(pool, list);
+	}
+}
+
+struct ldap_connection_pool *
+ldap_connection_pool_init(unsigned int max_connections)
+{
+	struct ldap_connection_pool *pool;
+
+	pool = i_new(struct ldap_connection_pool, 1);
+	pool->max_connections = max_connections;
+	return pool;
+}
+
+void ldap_connection_pool_deinit(struct ldap_connection_pool **_pool)
+{
+	struct ldap_connection_pool *pool = *_pool;
+
+	*_pool = NULL;
+
+	ldap_connection_pool_shrink_to(pool, 0);
+	i_assert(pool->conn_list == NULL);
+	i_free(pool);
+}
+
+int ldap_connection_pool_get(struct ldap_connection_pool *pool,
+			     struct ldap_client *client,
+			     const struct ldap_client_settings *set,
+			     struct ldap_connection_list **list_r,
+			     const char **error_r)
+{
+	struct ldap_connection_list *list;
+	struct ldap_connection *conn;
+
+	for (list = pool->conn_list; list != NULL; list = list->next) {
+		if (ldap_connection_have_settings(list->conn, set)) {
+			list->refcount++;
+			*list_r = list;
+			return 0;
+		}
+	}
+	if (ldap_connection_init(client, set, &conn, error_r) < 0)
+		return -1;
+
+	list = i_new(struct ldap_connection_list, 1);
+	list->conn = conn;
+	list->refcount++;
+
+	DLLIST_PREPEND(&pool->conn_list, list);
+	pool->conn_count++;
+
+	ldap_connection_pool_shrink_to(pool, pool->max_connections);
+	*list_r = list;
+	return 0;
+}
+
+void ldap_connection_pool_unref(struct ldap_connection_pool *pool,
+				struct ldap_connection_list **_list)
+{
+	struct ldap_connection_list *list = *_list;
+
+	*_list = NULL;
+
+	i_assert(list->refcount > 0);
+
+	if (--list->refcount == 0)
+		ldap_connection_pool_shrink_to(pool, pool->max_connections);
+}
+
+bool ldap_connection_pool_have_references(struct ldap_connection_pool *pool)
+{
+	struct ldap_connection_list *list;
+
+	for (list = pool->conn_list; list != NULL; list = list->next) {
+		if (list->refcount > 0)
+			return TRUE;
+	}
+	return FALSE;
+}
diff --git a/src/lib-ldap/ldap-connection-pool.h b/src/lib-ldap/ldap-connection-pool.h
new file mode 100644
index 00000000000..00cf1654eac
--- /dev/null
+++ b/src/lib-ldap/ldap-connection-pool.h
@@ -0,0 +1,27 @@
+#ifndef LDAP_CONNECTION_POOL_H
+#define LDAP_CONNECTION_POOL_H
+
+struct ldap_client;
+struct ldap_client_settings;
+
+struct ldap_connection_list {
+	struct ldap_connection_list *prev, *next;
+	struct ldap_connection *conn;
+	int refcount;
+};
+
+struct ldap_connection_pool *
+ldap_connection_pool_init(unsigned int max_connections);
+void ldap_connection_pool_deinit(struct ldap_connection_pool **_pool);
+/* Returns TRUE if there are connections with refcount>0 */
+bool ldap_connection_pool_have_references(struct ldap_connection_pool *pool);
+
+int ldap_connection_pool_get(struct ldap_connection_pool *pool,
+			     struct ldap_client *client,
+			     const struct ldap_client_settings *set,
+			     struct ldap_connection_list **list_r,
+			     const char **error_r);
+void ldap_connection_pool_unref(struct ldap_connection_pool *pool,
+				struct ldap_connection_list **list);
+
+#endif
diff --git a/src/lib-ldap/ldap-connection.c b/src/lib-ldap/ldap-connection.c
index 8a468bde44d..fada2cb361d 100644
--- a/src/lib-ldap/ldap-connection.c
+++ b/src/lib-ldap/ldap-connection.c
@@ -88,6 +88,41 @@ int ldap_connection_setup(struct ldap_connection *conn, const char **error_r)
 	return 0;
 }
 
+bool ldap_connection_have_settings(struct ldap_connection *conn,
+				   const struct ldap_client_settings *set)
+{
+	const struct ldap_client_settings *conn_set = &conn->set;
+
+	if (strcmp(conn_set->uri, set->uri) != 0)
+		return FALSE;
+	if (null_strcmp(conn_set->bind_dn, set->bind_dn) != 0)
+		return FALSE;
+	if (null_strcmp(conn_set->password, set->password) != 0)
+		return FALSE;
+	if (conn_set->timeout_secs != set->timeout_secs ||
+	    conn_set->max_idle_time_secs != set->max_idle_time_secs ||
+	    conn_set->debug != set->debug ||
+	    conn_set->require_ssl != set->require_ssl ||
+	    conn_set->start_tls != set->start_tls)
+		return FALSE;
+
+	if (set->ssl_set == NULL || !set->start_tls)
+		return TRUE;
+
+	/* check SSL settings */
+	if (null_strcmp(conn->ssl_set.protocols, set->ssl_set->protocols) != 0)
+		return FALSE;
+	if (null_strcmp(conn->ssl_set.cipher_list, set->ssl_set->cipher_list) != 0)
+		return FALSE;
+	if (null_strcmp(conn->ssl_set.ca_file, set->ssl_set->ca_file) != 0)
+		return FALSE;
+	if (null_strcmp(conn->ssl_set.cert, set->ssl_set->cert) != 0)
+		return FALSE;
+	if (null_strcmp(conn->ssl_set.key, set->ssl_set->key) != 0)
+		return FALSE;
+	return TRUE;
+}
+
 int ldap_connection_init(struct ldap_client *client,
 			 const struct ldap_client_settings *set,
 			 struct ldap_connection **conn_r, const char **error_r)
@@ -122,12 +157,15 @@ int ldap_connection_init(struct ldap_client *client,
 	conn->ssl_set.crypto_device = NULL;
 
 	if (set->ssl_set != NULL) {
+		/* keep in sync with ldap_connection_have_settings() */
+		conn->set.ssl_set = &conn->ssl_set;
 		conn->ssl_set.protocols = p_strdup(pool, set->ssl_set->protocols);
 		conn->ssl_set.cipher_list = p_strdup(pool, set->ssl_set->cipher_list);
 		conn->ssl_set.ca_file = p_strdup(pool, set->ssl_set->ca_file);
 		conn->ssl_set.cert = p_strdup(pool, set->ssl_set->cert);
 		conn->ssl_set.key = p_strdup(pool, set->ssl_set->key);
 	}
+	i_assert(ldap_connection_have_settings(conn, set));
 
 	if (ldap_connection_setup(conn, error_r) < 0) {
 		ldap_connection_deinit(&conn);
diff --git a/src/lib-ldap/ldap-private.h b/src/lib-ldap/ldap-private.h
index e6004db8b23..fa724f40df9 100644
--- a/src/lib-ldap/ldap-private.h
+++ b/src/lib-ldap/ldap-private.h
@@ -108,6 +108,8 @@ int ldap_connection_init(struct ldap_client *client,
 			 struct ldap_connection **conn_r, const char **error_r);
 void ldap_connection_deinit(struct ldap_connection **_conn);
 void ldap_connection_switch_ioloop(struct ldap_connection *conn);
+bool ldap_connection_have_settings(struct ldap_connection *conn,
+				   const struct ldap_client_settings *set);
 
 void ldap_connection_search_start(struct ldap_connection *conn,
 				  const struct ldap_search_input *input,

From 5f50fecfbf4eccdfea1550f1412b9790428cd4e4 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 3 May 2016 22:18:54 +0300
Subject: [PATCH 099/450] dsync: Fixed assert-crash if mailbox_attribute_set()
 fails

We shouldn't set importer->mail_error, because we're not going to
fail the import.

Fixes assert-crash:
dsync-mailbox-import.c: line 2812 (dsync_mailbox_import_deinit): assertion failed: (importer->failed == (importer->mail_error != 0))
---
 src/doveadm/dsync/dsync-mailbox-import.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c
index f561787dfb7..3392a653492 100644
--- a/src/doveadm/dsync/dsync-mailbox-import.c
+++ b/src/doveadm/dsync/dsync-mailbox-import.c
@@ -523,7 +523,7 @@ dsync_mailbox_import_attribute_real(struct dsync_mailbox_importer *importer,
 				  attr->key, &value) < 0) {
 		i_error("Mailbox %s: Failed to set attribute %s: %s",
 			mailbox_get_vname(importer->box), attr->key,
-			mailbox_get_last_error(importer->box, &importer->mail_error));
+			mailbox_get_last_error(importer->box, NULL));
 		/* the attributes aren't vital, don't fail everything just
 		   because of them. */
 	}

From 139007fda81b3f5027c79a892f8d329629b504c6 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 4 May 2016 15:36:44 +0300
Subject: [PATCH 100/450] imap: Set client_command_context.tagline_reply before
 syncing.

Command post-hooks can't see it otherwise.
---
 src/imap/imap-sync.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/imap/imap-sync.c b/src/imap/imap-sync.c
index df70625380c..5403605cf99 100644
--- a/src/imap/imap-sync.c
+++ b/src/imap/imap-sync.c
@@ -768,12 +768,13 @@ bool cmd_sync(struct client_command_context *cmd, enum mailbox_sync_flags flags,
 			client_send_tagline(cmd, tagline);
 		return TRUE;
 	}
+	cmd->tagline_reply = p_strdup(cmd->pool, tagline);
 
 	cmd->sync = p_new(cmd->pool, struct client_sync_context, 1);
 	cmd->sync->counter = client->sync_counter;
 	cmd->sync->flags = flags;
 	cmd->sync->imap_flags = imap_flags;
-	cmd->sync->tagline = p_strdup(cmd->pool, tagline);
+	cmd->sync->tagline = cmd->tagline_reply;
 	cmd->state = CLIENT_COMMAND_STATE_WAIT_SYNC;
 
 	cmd->func = NULL;

From eb239417679f4d5bdf0299db64e4ac31d63addda Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Mon, 2 May 2016 16:21:27 +0300
Subject: [PATCH 101/450] lib-ldap: Do not retry with bad credentials

---
 src/lib-ldap/ldap-connection.c | 33 ++++++++++++++++++++++++++++++---
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/src/lib-ldap/ldap-connection.c b/src/lib-ldap/ldap-connection.c
index fada2cb361d..76bab5fea87 100644
--- a/src/lib-ldap/ldap-connection.c
+++ b/src/lib-ldap/ldap-connection.c
@@ -16,6 +16,8 @@ static
 void ldap_connection_request_destroy(struct ldap_op_queue_entry **req);
 static
 int ldap_connection_connect(struct ldap_connection *conn);
+static
+void ldap_connection_send_next(struct ldap_connection *conn);
 
 void ldap_connection_deinit(struct ldap_connection **_conn)
 {
@@ -422,6 +424,28 @@ void ldap_connection_abort_request(struct ldap_op_queue_entry *req)
 	i_unreached();
 }
 
+static
+void ldap_connection_abort_all_requests(struct ldap_connection *conn)
+{
+	struct ldap_result res;
+	memset(&res, 0, sizeof(res));
+	res.openldap_ret = LDAP_TIMEOUT;
+	res.error_string = "Aborting LDAP requests due to failure";
+
+	unsigned int n = aqueue_count(conn->request_queue);
+	for (unsigned int i = 0; i < n; i++) {
+		struct ldap_op_queue_entry **reqp =
+			array_idx_modifiable(&(conn->request_array),
+		aqueue_idx(conn->request_queue, i));
+		if ((*reqp)->to_abort != NULL)
+			timeout_remove(&(*reqp)->to_abort);
+		if ((*reqp)->result_callback != NULL)
+			(*reqp)->result_callback(&res, (*reqp)->result_callback_ctx);
+		ldap_connection_request_destroy(reqp);
+	}
+	aqueue_clear(conn->request_queue);
+}
+
 static int
 ldap_connect_next_message(struct ldap_connection *conn,
 			  struct ldap_op_queue_entry *req, bool *finished_r)
@@ -589,8 +613,6 @@ ldap_connection_handle_message(struct ldap_connection *conn,
 	case LDAP_CONNECT_ERROR:
 #endif
 	case LDAP_UNAVAILABLE:
-		ldap_connection_kill(conn);
-		/* fall through */
 	case LDAP_OPERATIONS_ERROR:
 	case LDAP_BUSY:
 		/* requeue */
@@ -598,6 +620,12 @@ ldap_connection_handle_message(struct ldap_connection *conn,
 		ldap_connection_send_next(conn);
 		finished = FALSE;
 		break;
+	case LDAP_INVALID_CREDENTIALS: {
+		/* fail everything */
+		ldap_connection_kill(conn);
+		ldap_connection_abort_all_requests(conn);
+		return 0;
+	}
 	case LDAP_SIZELIMIT_EXCEEDED:
 	case LDAP_TIMELIMIT_EXCEEDED:
 	case LDAP_NO_SUCH_ATTRIBUTE:
@@ -613,7 +641,6 @@ ldap_connection_handle_message(struct ldap_connection *conn,
 	case LDAP_ALIAS_DEREF_PROBLEM:
 	case LDAP_FILTER_ERROR:
 	case LDAP_LOCAL_ERROR:
-	case LDAP_INVALID_CREDENTIALS:
 		finished = TRUE;
 		break;
 	default:

From 47f2320ca5ff396644e43e864d7b4ceb923f364a Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Mon, 2 May 2016 17:31:30 +0300
Subject: [PATCH 102/450] lib-ldap: Handle various LDAP failures correctly

---
 src/lib-ldap/ldap-connection.c | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/lib-ldap/ldap-connection.c b/src/lib-ldap/ldap-connection.c
index 76bab5fea87..e26ced40f73 100644
--- a/src/lib-ldap/ldap-connection.c
+++ b/src/lib-ldap/ldap-connection.c
@@ -57,6 +57,7 @@ int ldap_connection_setup(struct ldap_connection *conn, const char **error_r)
 	}
 
 	ldap_set_option(conn->conn, LDAP_OPT_X_TLS, &opt);
+	ldap_set_option(conn->conn, LDAP_OPT_X_TLS_REQUIRE_CERT, &opt);
 #ifdef LDAP_OPT_X_TLS_PROTOCOL_MIN
 	/* refuse to connect to SSLv2 as it's completely insecure */
 	opt = LDAP_OPT_X_TLS_PROTOCOL_SSL3;
@@ -339,14 +340,21 @@ ldap_connection_connect_parse(struct ldap_connection *conn,
 					"ldap_start_tls(uri=%s) failed: %s",
 					conn->set.uri, result_errmsg));
 				ldap_memfree(result_errmsg);
-				return LDAP_UNAVAILABLE; /* make sure it disconnects */
+				return LDAP_INVALID_CREDENTIALS; /* make sure it disconnects */
 			}
 		} else {
 			ret = ldap_parse_extended_result(conn->conn, message, &retoid, NULL, 0);
 			/* retoid can be NULL even if ret == 0 */
-			if (ret == 0 && retoid != NULL && strcmp(retoid, LDAP_EXOP_START_TLS) == 0) {
+			if (ret == 0) {
 				ret = ldap_install_tls(conn->conn);
-			} else if (ret == 0) ret = 2; /* make it fail on next if */
+				if (ret != 0) {
+					// if this fails we have to abort
+					ldap_connection_result_failure(conn, req, ret, t_strdup_printf(
+						"ldap_start_tls(uri=%s) failed: %s",
+						conn->set.uri, ldap_err2string(ret)));
+					return LDAP_INVALID_CREDENTIALS;
+				}
+			}
 			if (ret != LDAP_SUCCESS) {
 				if (conn->set.require_ssl) {
 					ldap_connection_result_failure(conn, req, ret, t_strdup_printf(

From 4497d28e19891c861e81e5bcb106f1f8e8b0facc Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 3 May 2016 15:27:22 +0300
Subject: [PATCH 103/450] lib-storage: Fixed potential crash in
 mailbox_sync_deinit() error handling

If mailbox_sync*() was called before mailbox was opened, the automatic
mailbox opening could fail. mailbox_sync_deinit() would still try to access
box->view, which would be NULL.
---
 src/lib-storage/index/index-sync.c | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/src/lib-storage/index/index-sync.c b/src/lib-storage/index/index-sync.c
index 0d4aba0ce4d..8b47937ae5e 100644
--- a/src/lib-storage/index/index-sync.c
+++ b/src/lib-storage/index/index-sync.c
@@ -291,6 +291,17 @@ void index_sync_update_recent_count(struct mailbox *box)
 	}
 }
 
+static void index_mailbox_sync_free(struct index_mailbox_sync_context *ctx)
+{
+	if (array_is_created(&ctx->flag_updates))
+		array_free(&ctx->flag_updates);
+	if (array_is_created(&ctx->hidden_updates))
+		array_free(&ctx->hidden_updates);
+	if (array_is_created(&ctx->all_flag_update_uids))
+		array_free(&ctx->all_flag_update_uids);
+	i_free(ctx);
+}
+
 int index_mailbox_sync_deinit(struct mailbox_sync_context *_ctx,
 			      struct mailbox_sync_status *status_r)
 {
@@ -314,6 +325,10 @@ int index_mailbox_sync_deinit(struct mailbox_sync_context *_ctx,
 			ret = -1;
 		}
 	}
+	if (ret < 0) {
+		index_mailbox_sync_free(ctx);
+		return -1;
+	}
 	index_mailbox_expunge_unseen_recent(ctx);
 
 	if ((_ctx->box->flags & MAILBOX_FLAG_DROP_RECENT) == 0 &&
@@ -327,18 +342,10 @@ int index_mailbox_sync_deinit(struct mailbox_sync_context *_ctx,
 
 	/* update search results after private index is updated */
 	index_sync_search_results_update(ctx);
-
-	if (array_is_created(&ctx->flag_updates))
-		array_free(&ctx->flag_updates);
-	if (array_is_created(&ctx->hidden_updates))
-		array_free(&ctx->hidden_updates);
-	if (array_is_created(&ctx->all_flag_update_uids))
-		array_free(&ctx->all_flag_update_uids);
-
 	/* update vsize header if wanted */
-	if (ret == 0)
-		index_mailbox_vsize_update_appends(_ctx->box);
-	i_free(ctx);
+	index_mailbox_vsize_update_appends(_ctx->box);
+
+	index_mailbox_sync_free(ctx);
 	return ret;
 }
 

From 81905b9300535941e918eb81760c26fc51edb704 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 3 May 2016 16:54:46 +0300
Subject: [PATCH 104/450] lib-mail: istream-header-filter - Check errors
 reading header

This probably doesn't affect the results much, since the stream_errno is set
anyway. But it's better to abort early, just in case the broken state might
end up asserting later.
---
 src/lib-mail/istream-header-filter.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/lib-mail/istream-header-filter.c b/src/lib-mail/istream-header-filter.c
index 1afb02641ba..d094f27dc1b 100644
--- a/src/lib-mail/istream-header-filter.c
+++ b/src/lib-mail/istream-header-filter.c
@@ -446,12 +446,12 @@ i_stream_header_filter_seek_to_header(struct header_filter_istream *mstream,
 	mstream->seen_eoh = FALSE;
 }
 
-static void skip_header(struct header_filter_istream *mstream)
+static int skip_header(struct header_filter_istream *mstream)
 {
 	size_t pos;
 
 	if (mstream->header_read)
-		return;
+		return 0;
 
 	if (mstream->istream.access_counter !=
 	    mstream->istream.parent->real_stream->access_counter) {
@@ -464,6 +464,7 @@ static void skip_header(struct header_filter_istream *mstream)
 		pos = i_stream_get_data_size(&mstream->istream.istream);
 		i_stream_skip(&mstream->istream.istream, pos);
 	}
+	return mstream->istream.istream.stream_errno != 0 ? -1 : 0;
 }
 
 static void
@@ -502,7 +503,8 @@ static void i_stream_header_filter_seek(struct istream_private *stream,
 	/* if we haven't parsed the whole header yet, we don't know if we
 	   want to seek inside header or body. so make sure we've parsed the
 	   header. */
-	skip_header(mstream);
+	if (skip_header(mstream) < 0)
+		return;
 	stream_reset_to(mstream, v_offset);
 
 	if (v_offset < mstream->header_size.virtual_size) {
@@ -542,7 +544,8 @@ i_stream_header_filter_stat(struct istream_private *stream, bool exact)
 
 	/* fix the filtered header size */
 	old_offset = stream->istream.v_offset;
-	skip_header(mstream);
+	if (skip_header(mstream) < 0)
+		return -1;
 
 	stream->statbuf.st_size -=
 		(off_t)mstream->header_size.physical_size -

From 03947494029d35847007ba8743076a9ad4fc07b0 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 3 May 2016 17:08:24 +0300
Subject: [PATCH 105/450] lib-mail: istream-header-filter: Fixed stat() with
 HEADER_FILTER_END_BODY_WITH_LF

---
 src/lib-mail/istream-header-filter.c      | 23 +++++++++++++++++++++++
 src/lib-mail/test-istream-header-filter.c | 19 +++++++++++++++++++
 2 files changed, 42 insertions(+)

diff --git a/src/lib-mail/istream-header-filter.c b/src/lib-mail/istream-header-filter.c
index d094f27dc1b..e7cae489ee5 100644
--- a/src/lib-mail/istream-header-filter.c
+++ b/src/lib-mail/istream-header-filter.c
@@ -547,6 +547,28 @@ i_stream_header_filter_stat(struct istream_private *stream, bool exact)
 	if (skip_header(mstream) < 0)
 		return -1;
 
+	if (!mstream->end_body_with_lf) {
+		/* no last-LF */
+	} else if (mstream->last_lf_added) {
+		/* yes, we have added LF */
+		stream->statbuf.st_size += mstream->crlf ? 2 : 1;
+	} else if (mstream->last_lf_offset != (uoff_t)-1) {
+		/* no, we didn't need to add LF */
+	} else {
+		/* check if we need to add LF */
+		i_stream_seek(stream->parent, st->st_size - 1);
+		(void)i_stream_read(stream->parent);
+		if (stream->parent->stream_errno != 0) {
+			stream->istream.stream_errno =
+				stream->parent->stream_errno;
+			return -1;
+		}
+		i_assert(stream->parent->eof);
+		ssize_t ret = handle_end_body_with_lf(mstream, -1);
+		if (ret > 0)
+			stream->statbuf.st_size += ret;
+	}
+
 	stream->statbuf.st_size -=
 		(off_t)mstream->header_size.physical_size -
 		(off_t)mstream->header_size.virtual_size;
@@ -601,6 +623,7 @@ i_stream_create_header_filter(struct istream *input,
 	mstream->add_missing_eoh = (flags & HEADER_FILTER_ADD_MISSING_EOH) != 0;
 	mstream->end_body_with_lf =
 		(flags & HEADER_FILTER_END_BODY_WITH_LF) != 0;
+	mstream->last_lf_offset = (uoff_t)-1;
 
 	mstream->istream.iostream.destroy = i_stream_header_filter_destroy;
 	mstream->istream.read = i_stream_header_filter_read;
diff --git a/src/lib-mail/test-istream-header-filter.c b/src/lib-mail/test-istream-header-filter.c
index cfa544ccb59..6c1357c5247 100644
--- a/src/lib-mail/test-istream-header-filter.c
+++ b/src/lib-mail/test-istream-header-filter.c
@@ -12,6 +12,7 @@ test_istream_run(struct istream *test_istream, struct istream *filter,
 		 unsigned int input_len, const char *output)
 {
 	unsigned int i, output_len = strlen(output);
+	const struct stat *st;
 	const unsigned char *data;
 	size_t size;
 
@@ -32,6 +33,8 @@ test_istream_run(struct istream *test_istream, struct istream *filter,
 	while (i_stream_read(filter) > 0) ;
 	data = i_stream_get_data(filter, &size);
 	test_assert(size == output_len && memcmp(data, output, size) == 0);
+	test_assert(i_stream_stat(filter, TRUE, &st) == 0 &&
+		    (uoff_t)st->st_size == size);
 }
 
 static void ATTR_NULL(3)
@@ -55,6 +58,7 @@ static void test_istream_filter(void)
 	unsigned int i, input_len = strlen(input);
 	unsigned int output_len = strlen(output);
 	const unsigned char *data;
+	const struct stat *st;
 	size_t size;
 
 	test_begin("i_stream_create_header_filter(exclude)");
@@ -86,11 +90,16 @@ static void test_istream_filter(void)
 	data = i_stream_get_data(filter, &size);
 	test_assert(size == output_len && memcmp(data, output, size) == 0);
 
+	test_assert(i_stream_stat(filter, TRUE, &st) == 0 &&
+		    (uoff_t)st->st_size == size);
+
 	i_stream_skip(filter, size);
 	i_stream_seek(filter, 0);
 	while (i_stream_read(filter) > 0) ;
 	data = i_stream_get_data(filter, &size);
 	test_assert(size == output_len && memcmp(data, output, size) == 0);
+	test_assert(i_stream_stat(filter, TRUE, &st) == 0 &&
+		    (uoff_t)st->st_size == size);
 
 	i_stream_unref(&filter);
 	i_stream_unref(&istream);
@@ -119,6 +128,7 @@ static void test_istream_filter_large_buffer(void)
 {
 	string_t *input, *output;
 	struct istream *istream, *filter;
+	const struct stat *st;
 	const unsigned char *data;
 	size_t size, prefix_len;
 	const char *p;
@@ -172,6 +182,9 @@ static void test_istream_filter_large_buffer(void)
 		i_assert(p != NULL);
 		test_assert(strcmp(p+1, str_c(output) + prefix_len) == 0);
 
+		test_assert(i_stream_stat(filter, TRUE, &st) == 0 &&
+			    (uoff_t)st->st_size == filter->v_offset + size);
+
 		/* seek back and retry once with caching and different
 		   buffer size */
 		i_stream_seek(filter, 0);
@@ -199,6 +212,7 @@ filter3_callback(struct header_filter_istream *input ATTR_UNUSED,
 static void test_istream_callbacks(void)
 {
 	string_t *input, *output;
+	const struct stat *st;
 	struct istream *istream, *filter;
 	unsigned int i;
 
@@ -230,6 +244,8 @@ static void test_istream_callbacks(void)
 			i_stream_skip(filter, i_stream_get_data_size(filter));
 	}
 
+	test_assert(i_stream_stat(filter, TRUE, &st) == 0 &&
+		    (uoff_t)st->st_size == str_len(output));
 	test_assert(strcmp(str_c(output), str_c(input)) == 0);
 	str_free(&input);
 	str_free(&output);
@@ -276,6 +292,7 @@ static void test_istream_end_body_with_lf(void)
 {
 	const char *input = "From: foo\n\nhello world";
 	const char *output = "From: foo\n\nhello world\n";
+	const struct stat *st;
 	struct istream *istream, *filter;
 	unsigned int i, input_len = strlen(input);
 	unsigned int output_len = strlen(output);
@@ -304,6 +321,8 @@ static void test_istream_end_body_with_lf(void)
 
 	data = i_stream_get_data(filter, &size);
 	test_assert(size == output_len && memcmp(data, output, size) == 0);
+	test_assert(i_stream_stat(filter, TRUE, &st) == 0 &&
+		    (uoff_t)st->st_size == filter->v_offset + size);
 
 	i_stream_skip(filter, size);
 	i_stream_seek(filter, 0);

From 7fd1292b9d55668d68ed5d046a1fec4cdd57b3e1 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 3 May 2016 17:25:07 +0300
Subject: [PATCH 106/450] lib-mail: istream-header-filter
 HEADER_FILTER_ADD_MISSING_EOH fixes

When using HEADER_FILTER_CRLF_PRESERVE, add CR to the EOH if the previous
header line ended with CRLF.

When header ends to a header without newline, add two newlines so we can get
the actual EOH added.
---
 src/lib-mail/istream-header-filter.c      | 13 +++++++-
 src/lib-mail/test-istream-header-filter.c | 37 +++++++++++++++++++++++
 2 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/src/lib-mail/istream-header-filter.c b/src/lib-mail/istream-header-filter.c
index e7cae489ee5..6b56356df30 100644
--- a/src/lib-mail/istream-header-filter.c
+++ b/src/lib-mail/istream-header-filter.c
@@ -38,6 +38,8 @@ struct header_filter_istream {
 	unsigned int add_missing_eoh:1;
 	unsigned int end_body_with_lf:1;
 	unsigned int last_lf_added:1;
+	unsigned int last_orig_crlf:1;
+	unsigned int last_added_newline:1;
 	unsigned int eoh_not_matched:1;
 	unsigned int prev_matched:1;
 };
@@ -134,6 +136,8 @@ static void add_eol(struct header_filter_istream *mstream, bool orig_crlf)
 		buffer_append(mstream->hdr_buf, "\r\n", 2);
 	else
 		buffer_append_c(mstream->hdr_buf, '\n');
+	mstream->last_orig_crlf = orig_crlf;
+	mstream->last_added_newline = TRUE;
 }
 
 static ssize_t hdr_stream_update_pos(struct header_filter_istream *mstream)
@@ -285,6 +289,11 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 		if (mstream->hdr_buf->used >= mstream->istream.max_buffer_size)
 			break;
 	}
+	if (mstream->hdr_buf->used > 0) {
+		const unsigned char *data = mstream->hdr_buf->data;
+		mstream->last_added_newline =
+			data[mstream->hdr_buf->used-1] == '\n';
+	}
 
 	if (hdr_ret < 0) {
 		if (mstream->istream.parent->stream_errno != 0) {
@@ -296,7 +305,9 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 		}
 		if (!mstream->seen_eoh && mstream->add_missing_eoh) {
 			mstream->seen_eoh = TRUE;
-			add_eol(mstream, FALSE);
+			if (!mstream->last_added_newline)
+				add_eol(mstream, mstream->last_orig_crlf);
+			add_eol(mstream, mstream->last_orig_crlf);
 		}
 	}
 
diff --git a/src/lib-mail/test-istream-header-filter.c b/src/lib-mail/test-istream-header-filter.c
index 6c1357c5247..0f92ecbfd1b 100644
--- a/src/lib-mail/test-istream-header-filter.c
+++ b/src/lib-mail/test-istream-header-filter.c
@@ -349,6 +349,42 @@ static void test_istream_end_body_with_lf(void)
 	test_end();
 }
 
+static void test_istream_add_missing_eoh(void)
+{
+	struct {
+		const char *input;
+		const char *output;
+		unsigned int extra;
+	} tests[] = {
+		{ "From: foo", "From: foo\n\n", 1 },
+		{ "From: foo\n", "From: foo\n\n", 1 },
+		{ "From: foo\n\n", "From: foo\n\n", 1 },
+		{ "From: foo\n\nbar", "From: foo\n\nbar", 0 },
+		{ "From: foo\r\n", "From: foo\r\n\r\n", 1 },
+		{ "From: foo\r\n\r\n", "From: foo\r\n\r\n", 0 },
+		{ "From: foo\r\n\r\nbar", "From: foo\r\n\r\nbar", 0 }
+	};
+	struct istream *istream, *filter;
+	unsigned int i;
+
+	test_begin("i_stream_create_header_filter(add_missing_eoh)");
+	for (i = 0; i < N_ELEMENTS(tests); i++) {
+		istream = test_istream_create(tests[i].input);
+		filter = i_stream_create_header_filter(istream,
+					       HEADER_FILTER_EXCLUDE |
+					       HEADER_FILTER_CRLF_PRESERVE |
+					       HEADER_FILTER_ADD_MISSING_EOH,
+					       NULL, 0,
+					       *null_header_filter_callback, (void *)NULL);
+		test_istream_run(istream, filter,
+				 strlen(tests[i].input) + tests[i].extra,
+				 tests[i].output);
+		i_stream_unref(&filter);
+		i_stream_unref(&istream);
+	}
+	test_end();
+}
+
 static void ATTR_NULL(3)
 strip_eoh_callback(struct header_filter_istream *input ATTR_UNUSED,
 		   struct message_header_line *hdr,
@@ -383,6 +419,7 @@ int main(void)
 		test_istream_filter_large_buffer,
 		test_istream_callbacks,
 		test_istream_edit,
+		test_istream_add_missing_eoh,
 		test_istream_end_body_with_lf,
 		test_istream_strip_eoh,
 		NULL

From 71c09f6b25467dfb6ae79dcee371781d2532d2d2 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 3 May 2016 17:38:34 +0300
Subject: [PATCH 107/450] lib-mail: istream-header-filter: Fixed stat() with
 HEADER_FILTER_HIDE_BODY

---
 src/lib-mail/istream-header-filter.c      |  5 ++-
 src/lib-mail/test-istream-header-filter.c | 37 +++++++++++++++++++++++
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/src/lib-mail/istream-header-filter.c b/src/lib-mail/istream-header-filter.c
index 6b56356df30..871068a8a53 100644
--- a/src/lib-mail/istream-header-filter.c
+++ b/src/lib-mail/istream-header-filter.c
@@ -558,7 +558,10 @@ i_stream_header_filter_stat(struct istream_private *stream, bool exact)
 	if (skip_header(mstream) < 0)
 		return -1;
 
-	if (!mstream->end_body_with_lf) {
+	if (mstream->hide_body) {
+		/* no body */
+		stream->statbuf.st_size = mstream->header_size.physical_size;
+	} else if (!mstream->end_body_with_lf) {
 		/* no last-LF */
 	} else if (mstream->last_lf_added) {
 		/* yes, we have added LF */
diff --git a/src/lib-mail/test-istream-header-filter.c b/src/lib-mail/test-istream-header-filter.c
index 0f92ecbfd1b..9ad9ffbe5e5 100644
--- a/src/lib-mail/test-istream-header-filter.c
+++ b/src/lib-mail/test-istream-header-filter.c
@@ -385,6 +385,42 @@ static void test_istream_add_missing_eoh(void)
 	test_end();
 }
 
+static void test_istream_hide_body(void)
+{
+	struct {
+		const char *input;
+		const char *output;
+		int extra;
+	} tests[] = {
+		{ "From: foo", "From: foo", 0 },
+		{ "From: foo\n", "From: foo\n", 0 },
+		{ "From: foo\n\n", "From: foo\n\n", 1 },
+		{ "From: foo\n\nbar", "From: foo\n\n", -2 },
+		{ "From: foo\r\n", "From: foo\r\n", 0 },
+		{ "From: foo\r\n\r\n", "From: foo\r\n\r\n", 0 },
+		{ "From: foo\r\n\r\nbar", "From: foo\r\n\r\n", -3 }
+	};
+	struct istream *istream, *filter;
+	unsigned int i;
+
+	test_begin("i_stream_create_header_filter(add_missing_eoh)");
+	for (i = 0; i < N_ELEMENTS(tests); i++) {
+		istream = test_istream_create(tests[i].input);
+		filter = i_stream_create_header_filter(istream,
+					       HEADER_FILTER_EXCLUDE |
+					       HEADER_FILTER_CRLF_PRESERVE |
+					       HEADER_FILTER_HIDE_BODY,
+					       NULL, 0,
+					       *null_header_filter_callback, (void *)NULL);
+		test_istream_run(istream, filter,
+				 strlen(tests[i].input) + tests[i].extra,
+				 tests[i].output);
+		i_stream_unref(&filter);
+		i_stream_unref(&istream);
+	}
+	test_end();
+}
+
 static void ATTR_NULL(3)
 strip_eoh_callback(struct header_filter_istream *input ATTR_UNUSED,
 		   struct message_header_line *hdr,
@@ -421,6 +457,7 @@ int main(void)
 		test_istream_edit,
 		test_istream_add_missing_eoh,
 		test_istream_end_body_with_lf,
+		test_istream_hide_body,
 		test_istream_strip_eoh,
 		NULL
 	};

From 90c3923258819fcc222955fb71f0e5d77387c754 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 3 May 2016 18:25:50 +0300
Subject: [PATCH 108/450] lib-mail: istream-header-filter - fixed adding
 headers at eoh

---
 src/lib-mail/istream-header-filter.c      |  2 +-
 src/lib-mail/test-istream-header-filter.c | 11 +++++++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/lib-mail/istream-header-filter.c b/src/lib-mail/istream-header-filter.c
index 871068a8a53..728ee5d2b63 100644
--- a/src/lib-mail/istream-header-filter.c
+++ b/src/lib-mail/istream-header-filter.c
@@ -200,7 +200,7 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 		if (hdr->eoh) {
 			mstream->seen_eoh = TRUE;
 			matched = TRUE;
-			if (mstream->header_parsed) {
+			if (mstream->header_parsed && !mstream->headers_edited) {
 				if (mstream->eoh_not_matched)
 					matched = !matched;
 			} else if (mstream->callback != NULL) {
diff --git a/src/lib-mail/test-istream-header-filter.c b/src/lib-mail/test-istream-header-filter.c
index 9ad9ffbe5e5..7d338c693dc 100644
--- a/src/lib-mail/test-istream-header-filter.c
+++ b/src/lib-mail/test-istream-header-filter.c
@@ -260,7 +260,14 @@ edit_callback(struct header_filter_istream *input,
 	      struct message_header_line *hdr,
 	      bool *matched, void *context ATTR_UNUSED)
 {
-	if (hdr != NULL && strcasecmp(hdr->name, "To") == 0) {
+	if (hdr == NULL)
+		return;
+	if (hdr->eoh) {
+		/* add a new header */
+		const char *new_hdr = "Added: header\n\n";
+		i_stream_header_filter_add(input, new_hdr, strlen(new_hdr));
+		*matched = FALSE;
+	} else if (strcasecmp(hdr->name, "To") == 0) {
 		/* modify To header */
 		const char *new_to = "To: 123\n";
 		*matched = TRUE;
@@ -271,7 +278,7 @@ edit_callback(struct header_filter_istream *input,
 static void test_istream_edit(void)
 {
 	const char *input = "From: foo\nTo: bar\n\nhello world\n";
-	const char *output = "From: foo\nTo: 123\n\nhello world\n";
+	const char *output = "From: foo\nTo: 123\nAdded: header\n\nhello world\n";
 	struct istream *istream, *filter;
 
 	test_begin("i_stream_create_header_filter(edit)");

From 1c3f119d72701fea1f5bf65cca17ace1d8026ef9 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 2 May 2016 23:43:15 +0300
Subject: [PATCH 109/450] lib-storage: Make it easier to debug mail's unclosed
 istream

---
 src/lib-storage/index/index-mail.c | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c
index dceb2c949bb..856822e71f3 100644
--- a/src/lib-storage/index/index-mail.c
+++ b/src/lib-storage/index/index-mail.c
@@ -1499,6 +1499,8 @@ static void index_mail_close_streams_full(struct index_mail *mail, bool closing)
 	if (data->filter_stream != NULL)
 		i_stream_unref(&data->filter_stream);
 	if (data->stream != NULL) {
+		struct istream *orig_stream = data->stream;
+
 		data->destroying_stream = TRUE;
 		if (!closing && data->destroy_callback_set) {
 			/* we're replacing the stream with a new one. it's
@@ -1508,12 +1510,13 @@ static void index_mail_close_streams_full(struct index_mail *mail, bool closing)
 				index_mail_stream_destroy_callback);
 		}
 		i_stream_unref(&data->stream);
- 		if (closing) {
-			/* there must be no references to the mail when the
-			   mail is being closed. */
-			i_assert(!mail->data.destroying_stream);
-		} else {
+		/* there must be no references to the mail when the
+		   mail is being closed. */
+		if (!closing)
 			data->destroying_stream = FALSE;
+		else if (mail->data.destroying_stream) {
+			i_panic("Input stream %s unexpectedly has references",
+				i_stream_get_name(orig_stream));
 		}
 
 		data->initialized_wrapper_stream = FALSE;

From 9233125a5524ca2dfe324de19d95576f8a360562 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 3 May 2016 20:20:28 +0300
Subject: [PATCH 110/450] doveadm who: Fixed listing LMTP sessions

---
 src/doveadm/doveadm-who.c | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/src/doveadm/doveadm-who.c b/src/doveadm/doveadm-who.c
index 0fd7d94c2d0..3b75b9a9a69 100644
--- a/src/doveadm/doveadm-who.c
+++ b/src/doveadm/doveadm-who.c
@@ -59,6 +59,8 @@ static int who_parse_line(const char *line, struct who_line *line_r)
 
 	memset(line_r, 0, sizeof(*line_r));
 
+	/* ident = service/ip/username (imap, pop3)
+	   or      service/username (lmtp) */
 	p = strchr(ident, '/');
 	if (p == NULL)
 		return -1;
@@ -66,12 +68,15 @@ static int who_parse_line(const char *line, struct who_line *line_r)
 		return -1;
 	line_r->service = t_strdup_until(ident, p++);
 	line_r->username = strchr(p, '/');
-	if (line_r->username == NULL)
-		return -1;
+	if (line_r->username == NULL) {
+		/* no IP */
+		line_r->username = p;
+	} else {
+		ip_str = t_strdup_until(p, line_r->username++);
+		(void)net_addr2ip(ip_str, &line_r->ip);
+	}
 	if (str_to_uint(refcount_str, &line_r->refcount) < 0)
 		return -1;
-	ip_str = t_strdup_until(p, line_r->username++);
-	(void)net_addr2ip(ip_str, &line_r->ip);
 	return 0;
 }
 

From b0626c087fd1a15d47790873cae746324211e305 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 3 May 2016 20:28:42 +0300
Subject: [PATCH 111/450] lib-mail: Fixed istream-header-filter unit test names

---
 src/lib-mail/test-istream-header-filter.c | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/lib-mail/test-istream-header-filter.c b/src/lib-mail/test-istream-header-filter.c
index 7d338c693dc..0523390d8cd 100644
--- a/src/lib-mail/test-istream-header-filter.c
+++ b/src/lib-mail/test-istream-header-filter.c
@@ -61,7 +61,7 @@ static void test_istream_filter(void)
 	const struct stat *st;
 	size_t size;
 
-	test_begin("i_stream_create_header_filter(exclude)");
+	test_begin("i_stream_create_header_filter: exclude");
 	istream = test_istream_create(input);
 	filter = i_stream_create_header_filter(istream,
 					       HEADER_FILTER_EXCLUDE |
@@ -134,7 +134,7 @@ static void test_istream_filter_large_buffer(void)
 	const char *p;
 	unsigned int i;
 
-	test_begin("i_stream_create_header_filter(large buffer)");
+	test_begin("i_stream_create_header_filter: large buffer");
 
 	input = str_new(default_pool, 1024*128);
 	output = str_new(default_pool, 1024*128);
@@ -216,7 +216,7 @@ static void test_istream_callbacks(void)
 	struct istream *istream, *filter;
 	unsigned int i;
 
-	test_begin("i_stream_create_header_filter(callbacks)");
+	test_begin("i_stream_create_header_filter: callbacks");
 
 	input = str_new(default_pool, 1024*128);
 	output = str_new(default_pool, 1024*128);
@@ -281,7 +281,7 @@ static void test_istream_edit(void)
 	const char *output = "From: foo\nTo: 123\nAdded: header\n\nhello world\n";
 	struct istream *istream, *filter;
 
-	test_begin("i_stream_create_header_filter(edit)");
+	test_begin("i_stream_create_header_filter: edit headers");
 	istream = test_istream_create(input);
 	filter = i_stream_create_header_filter(istream,
 					       HEADER_FILTER_EXCLUDE |
@@ -307,7 +307,7 @@ static void test_istream_end_body_with_lf(void)
 	string_t *str = t_str_new(64);
 	size_t size;
 
-	test_begin("i_stream_create_header_filter(end_body_with_lf)");
+	test_begin("i_stream_create_header_filter: end_body_with_lf");
 	istream = test_istream_create(input);
 	filter = i_stream_create_header_filter(istream,
 					       HEADER_FILTER_EXCLUDE |
@@ -374,7 +374,7 @@ static void test_istream_add_missing_eoh(void)
 	struct istream *istream, *filter;
 	unsigned int i;
 
-	test_begin("i_stream_create_header_filter(add_missing_eoh)");
+	test_begin("i_stream_create_header_filter: add missing EOH");
 	for (i = 0; i < N_ELEMENTS(tests); i++) {
 		istream = test_istream_create(tests[i].input);
 		filter = i_stream_create_header_filter(istream,
@@ -410,7 +410,7 @@ static void test_istream_hide_body(void)
 	struct istream *istream, *filter;
 	unsigned int i;
 
-	test_begin("i_stream_create_header_filter(add_missing_eoh)");
+	test_begin("i_stream_create_header_filter: hide body");
 	for (i = 0; i < N_ELEMENTS(tests); i++) {
 		istream = test_istream_create(tests[i].input);
 		filter = i_stream_create_header_filter(istream,
@@ -443,7 +443,7 @@ static void test_istream_strip_eoh(void)
 	const char *output = "From: foo\nTo: bar\nhello world\n";
 	struct istream *istream, *filter;
 
-	test_begin("i_stream_create_header_filter(edit)");
+	test_begin("i_stream_create_header_filter: strip_eoh");
 	istream = test_istream_create(input);
 	filter = i_stream_create_header_filter(istream,
 			HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR, NULL, 0,

From e36e97499078ff53671e00de863379b89823368c Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 4 May 2016 18:34:59 +0300
Subject: [PATCH 112/450] lib-fs: fs-metawrap stat() error handling
 improvement.

We can't just treat i_stream_get_size() returning 0 as the reason being
istream is async and not fully read. It might be, but it might also be
because of other reasons. And since we're closing the istream we couldn't
even properly finish up the async handling. So for now just return an
error if we see this happening.
---
 src/lib-fs/fs-metawrap.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/lib-fs/fs-metawrap.c b/src/lib-fs/fs-metawrap.c
index 0e9f09f70d7..9785ee7f43c 100644
--- a/src/lib-fs/fs-metawrap.c
+++ b/src/lib-fs/fs-metawrap.c
@@ -503,7 +503,10 @@ static int fs_metawrap_stat(struct fs_file *_file, struct stat *st_r)
 	}
 	i_stream_unref(&input);
 	if (ret == 0) {
-		fs_set_error_async(_file->fs);
+		/* we shouldn't get here */
+		fs_set_error(_file->fs, "i_stream_get_size(%s) returned size as unknown",
+			     fs_file_path(_file));
+		errno = EIO;
 		return -1;
 	}
 

From bb3679aad440b7a971fe4c9f53fbb6f5fd29bb50 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 5 May 2016 14:58:46 +0300
Subject: [PATCH 113/450] stats: Fixed ADD-USER stats tracking (auth stats)

---
 src/stats/mail-user.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/stats/mail-user.c b/src/stats/mail-user.c
index 3358483fa76..6e0f7b87c62 100644
--- a/src/stats/mail-user.c
+++ b/src/stats/mail-user.c
@@ -122,7 +122,7 @@ void mail_user_refresh(struct mail_user *user,
 int mail_user_add_parse(const char *const *args, const char **error_r)
 {
 	struct mail_user *user;
-	struct stats *diff_stats;
+	struct stats *empty_stats, *diff_stats;
 	buffer_t *buf;
 	const char *service, *error;
 
@@ -141,8 +141,9 @@ int mail_user_add_parse(const char *const *args, const char **error_r)
 					   user->name, service);
 		return -1;
 	}
+	empty_stats = stats_alloc(pool_datastack_create());
 	diff_stats = stats_alloc(pool_datastack_create());
-	if (!stats_import(buf->data, buf->used, user->stats, diff_stats, &error)) {
+	if (!stats_import(buf->data, buf->used, empty_stats, diff_stats, &error)) {
 		*error_r = t_strdup_printf("ADD-USER %s %s: %s",
 					   user->name, service, error);
 		return -1;

From 2615930e573084cac91dde95ab2052a267ae90e7 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 5 May 2016 16:15:15 +0300
Subject: [PATCH 114/450] lib-fs: Fixes to stats count tracking

fs_exists(), fs_copy(), fs_rename() and fs_delete() could have increased the
count multiple times on async operations.
---
 src/lib-fs/fs-api.c | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c
index b50fbe0f53c..2655b33d4c9 100644
--- a/src/lib-fs/fs-api.c
+++ b/src/lib-fs/fs-api.c
@@ -749,13 +749,14 @@ int fs_exists(struct fs_file *file)
 		else
 			return errno == ENOENT ? 0 : -1;
 	}
-	file->fs->stats.exists_count++;
 	fs_file_timing_start(file, FS_OP_EXISTS);
 	T_BEGIN {
 		ret = file->fs->v.exists(file);
 	} T_END;
-	if (!(ret < 0 && errno == EAGAIN))
+	if (!(ret < 0 && errno == EAGAIN)) {
+		file->fs->stats.exists_count++;
 		fs_file_timing_end(file, FS_OP_EXISTS);
+	}
 	return ret;
 }
 
@@ -843,13 +844,13 @@ int fs_copy(struct fs_file *src, struct fs_file *dest)
 		return -1;
 	}
 
-	dest->fs->stats.copy_count++;
 	fs_file_timing_start(dest, FS_OP_COPY);
 	T_BEGIN {
 		ret = src->fs->v.copy(src, dest);
 	} T_END;
 	if (!(ret < 0 && errno == EAGAIN)) {
 		fs_file_timing_end(dest, FS_OP_COPY);
+		dest->fs->stats.copy_count++;
 		dest->metadata_changed = FALSE;
 	}
 	return ret;
@@ -864,6 +865,7 @@ int fs_copy_finish_async(struct fs_file *dest)
 	} T_END;
 	if (!(ret < 0 && errno == EAGAIN)) {
 		fs_file_timing_end(dest, FS_OP_COPY);
+		dest->fs->stats.copy_count++;
 		dest->metadata_changed = FALSE;
 	}
 	return ret;
@@ -875,13 +877,14 @@ int fs_rename(struct fs_file *src, struct fs_file *dest)
 
 	i_assert(src->fs == dest->fs);
 
-	dest->fs->stats.rename_count++;
 	fs_file_timing_start(dest, FS_OP_RENAME);
 	T_BEGIN {
 		ret = src->fs->v.rename(src, dest);
 	} T_END;
-	if (!(ret < 0 && errno == EAGAIN))
+	if (!(ret < 0 && errno == EAGAIN)) {
+		dest->fs->stats.rename_count++;
 		fs_file_timing_end(dest, FS_OP_RENAME);
+	}
 	return ret;
 }
 
@@ -889,13 +892,14 @@ int fs_delete(struct fs_file *file)
 {
 	int ret;
 
-	file->fs->stats.delete_count++;
 	fs_file_timing_start(file, FS_OP_DELETE);
 	T_BEGIN {
 		ret = file->fs->v.delete_file(file);
 	} T_END;
-	if (!(ret < 0 && errno == EAGAIN))
+	if (!(ret < 0 && errno == EAGAIN)) {
+		file->fs->stats.delete_count++;
 		fs_file_timing_end(file, FS_OP_DELETE);
+	}
 	return ret;
 }
 

From 501231e8455297016fdca992518a31b0904fc8b7 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 5 May 2016 18:10:46 +0300
Subject: [PATCH 115/450] lazy-expunge: Handle mailbox create race conditions.

Don't log an error if another process just created the lazy-expunge mailbox.
---
 src/plugins/lazy-expunge/lazy-expunge-plugin.c | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/plugins/lazy-expunge/lazy-expunge-plugin.c b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
index cce392dac53..e69e372bcb8 100644
--- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c
+++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
@@ -125,13 +125,19 @@ mailbox_open_or_create(struct mailbox_list *list, struct mailbox *src_box,
 	}
 
 	/* try creating and re-opening it. */
-	if (mailbox_create(box, NULL, FALSE) < 0 ||
-	    mailbox_open(box) < 0) {
+	if (mailbox_create(box, NULL, FALSE) < 0 &&
+	    mailbox_get_last_mail_error(box) != MAIL_ERROR_EXISTS) {
 		*error_r = t_strdup_printf("Failed to create mailbox %s: %s", name,
 					   mailbox_get_last_error(box, NULL));
 		mailbox_free(&box);
 		return NULL;
 	}
+	if (mailbox_open(box) < 0) {
+		*error_r = t_strdup_printf("Failed to open created mailbox %s: %s", name,
+					   mailbox_get_last_error(box, NULL));
+		mailbox_free(&box);
+		return NULL;
+	}
 	return box;
 }
 

From 4a92e7afb9683c8c4de60a822223c2ad6e0e55bd Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 6 May 2016 14:22:29 +0300
Subject: [PATCH 116/450] lib: Improved connection_disconnect_reason()

---
 src/lib/connection.c | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/src/lib/connection.c b/src/lib/connection.c
index f9fab8f798b..b73a8c32412 100644
--- a/src/lib/connection.c
+++ b/src/lib/connection.c
@@ -359,7 +359,26 @@ int connection_input_read(struct connection *conn)
 
 const char *connection_disconnect_reason(struct connection *conn)
 {
-	return io_stream_get_disconnect_reason(conn->input, conn->output);
+	switch (conn->disconnect_reason) {
+	case CONNECTION_DISCONNECT_DEINIT:
+		return "Deinitializing";
+	case CONNECTION_DISCONNECT_CONNECT_TIMEOUT: {
+		unsigned int msecs =
+			conn->list->set.client_connect_timeout_msecs;
+		return t_strdup_printf("connect() timed out in %u.%03u secs",
+				       msecs/1000, msecs%1000);
+	}
+	case CONNECTION_DISCONNECT_IDLE_TIMEOUT:
+		return "Idle timeout";
+	case CONNECTION_DISCONNECT_CONN_CLOSED:
+		if (conn->input == NULL)
+			return t_strdup_printf("connect() failed: %m");
+		/* fall through */
+	case CONNECTION_DISCONNECT_NOT:
+	case CONNECTION_DISCONNECT_BUFFER_FULL:
+		return io_stream_get_disconnect_reason(conn->input, conn->output);
+	}
+	i_unreached();
 }
 
 void connection_switch_ioloop(struct connection *conn)

From 0e65dc9c8a938fde37dea36d5e17dc7ea522d24f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 6 May 2016 14:34:57 +0300
Subject: [PATCH 117/450] lib: Added i_stream_get_disconnect_reason()

---
 src/lib/istream.c | 5 +++++
 src/lib/istream.h | 4 ++++
 2 files changed, 9 insertions(+)

diff --git a/src/lib/istream.c b/src/lib/istream.c
index 2d70a230db7..d525c8b7701 100644
--- a/src/lib/istream.c
+++ b/src/lib/istream.c
@@ -97,6 +97,11 @@ const char *i_stream_get_error(struct istream *stream)
 	return strerror(stream->stream_errno);
 }
 
+const char *i_stream_get_disconnect_reason(struct istream *stream)
+{
+	return io_stream_get_disconnect_reason(stream, NULL);
+}
+
 void i_stream_close(struct istream *stream)
 {
 	i_stream_close_full(stream, TRUE);
diff --git a/src/lib/istream.h b/src/lib/istream.h
index 0c96e989671..025afd0f54c 100644
--- a/src/lib/istream.h
+++ b/src/lib/istream.h
@@ -77,6 +77,10 @@ int i_stream_get_fd(struct istream *stream);
 /* Returns error string for the last error. It also returns "EOF" in case there
    is no error, but eof is set. Otherwise it returns "<no error>". */
 const char *i_stream_get_error(struct istream *stream);
+/* Returns human-readable reason for why istream was disconnected. This can be
+   called to log the error when i_stream_read() returns -1. If there's an error
+   the output is identical to i_stream_get_error(). */
+const char *i_stream_get_disconnect_reason(struct istream *stream);
 
 /* Mark the stream and all of its parent streams closed. Any reads after this
    will return -1. The data already read can still be used. */

From fc0f22ebd30428111592e0a7e404794c6d71f20f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 3 May 2016 17:43:00 +0300
Subject: [PATCH 118/450] auth: Makefile dependency fix

Hopefully fixes:
mv: cannot stat '.deps/auth-stats.Tpo': No such file or directory
---
 src/auth/Makefile.am | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/auth/Makefile.am b/src/auth/Makefile.am
index 3e00e246625..0fbc56de0bf 100644
--- a/src/auth/Makefile.am
+++ b/src/auth/Makefile.am
@@ -50,6 +50,7 @@ libpassword_a_SOURCES = \
 	password-scheme-rpa.c
 
 auth_libs = \
+	libstats_auth.la \
 	libpassword.a \
 	../lib-ntlm/libntlm.a \
 	../lib-otp/libotp.a \
@@ -74,7 +75,6 @@ auth_SOURCES = \
 	auth-request-stats.c \
 	auth-request-var-expand.c \
 	auth-settings.c \
-	auth-stats.c \
 	auth-fields.c \
 	auth-token.c \
 	auth-worker-client.c \
@@ -204,9 +204,9 @@ stats_moduledir = $(moduledir)/stats
 stats_module_LTLIBRARIES = libstats_auth.la
 
 libstats_auth_la_LDFLAGS = -module -avoid-version
-libstats_auth_la_LIBADD = auth-stats.lo $(LIBDOVECOT)
-libstats_auth_la_DEPENDENCIES = auth-stats.lo
-libstats_auth_la_SOURCES =
+libstats_auth_la_LIBADD = $(LIBDOVECOT)
+libstats_auth_la_DEPENDENCIES = $(LIBDOVECOT_DEPS)
+libstats_auth_la_SOURCES = auth-stats.c
 
 test_programs = \
 	test-auth-cache \

From 37f0b04489ab73227ae28678d6f15067adea1a94 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Fri, 6 May 2016 12:51:28 +0300
Subject: [PATCH 119/450] dict-ldap: Properly duplicate result

---
 src/lib-dict-extra/dict-ldap.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/lib-dict-extra/dict-ldap.c b/src/lib-dict-extra/dict-ldap.c
index b9d19dd5d07..d199ac8e1ed 100644
--- a/src/lib-dict-extra/dict-ldap.c
+++ b/src/lib-dict-extra/dict-ldap.c
@@ -260,7 +260,9 @@ static
 void ldap_dict_lookup_done(const struct dict_lookup_result *result, void *ctx)
 {
 	struct dict_lookup_result *res = ctx;
-	*res = *result;
+	res->ret = result->ret;
+	res->value = t_strdup(result->value);
+	res->error = t_strdup(result->error);
 }
 
 static void

From c8d8cb57937a065090d6e0bfe96425de79f69244 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 5 May 2016 21:23:17 +0300
Subject: [PATCH 120/450] stats: Work around shrinking system CPU usage

Happening at least in Debian's Linux kernel 4.3.0-1-amd64.

getrusage() may returns ru_stime = 4000 or 8000, but later it drops to 0.
We'll just work around this by switching to the previous working ru_stime.

This fixes errors like:
Error: stats: session stats shrank: sys_cpu 0.0 < 0.4000
---
 src/plugins/stats/mail-stats-fill.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/plugins/stats/mail-stats-fill.c b/src/plugins/stats/mail-stats-fill.c
index 7c48f6feab0..95e4445f6cb 100644
--- a/src/plugins/stats/mail-stats-fill.c
+++ b/src/plugins/stats/mail-stats-fill.c
@@ -1,6 +1,7 @@
 /* Copyright (c) 2011-2016 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
+#include "time-util.h"
 #include "stats-plugin.h"
 #include "mail-stats.h"
 
@@ -110,12 +111,19 @@ user_trans_stats_get(struct stats_user *suser, struct mail_stats *dest_r)
 
 void mail_stats_fill(struct stats_user *suser, struct mail_stats *stats_r)
 {
+	static struct rusage prev_usage;
 	struct rusage usage;
 
 	memset(stats_r, 0, sizeof(*stats_r));
 	/* cputime */
 	if (getrusage(RUSAGE_SELF, &usage) < 0)
 		memset(&usage, 0, sizeof(usage));
+	if (timeval_diff_usecs(&usage.ru_stime, &prev_usage.ru_stime) < 0) {
+		/* This seems to be a Linux bug. */
+		usage.ru_stime = prev_usage.ru_stime;
+	}
+	prev_usage = usage;
+
 	stats_r->user_cpu = usage.ru_utime;
 	stats_r->sys_cpu = usage.ru_stime;
 	stats_r->min_faults = usage.ru_minflt;

From 99e68380dfbc17b78c1cda139093dc108fa4f097 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 5 May 2016 21:26:37 +0300
Subject: [PATCH 121/450] stats: Handle getrusage() errors better

I'm not aware of these errors actually happening anywhere, but its
error handling wouldn't have been correct if previous getrusage() calls
had succeeded. Now if it fails, log an error once and just keep on using
the last working rusage.
---
 src/plugins/stats/mail-stats-fill.c | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/plugins/stats/mail-stats-fill.c b/src/plugins/stats/mail-stats-fill.c
index 95e4445f6cb..3c8ff360e4f 100644
--- a/src/plugins/stats/mail-stats-fill.c
+++ b/src/plugins/stats/mail-stats-fill.c
@@ -111,14 +111,19 @@ user_trans_stats_get(struct stats_user *suser, struct mail_stats *dest_r)
 
 void mail_stats_fill(struct stats_user *suser, struct mail_stats *stats_r)
 {
+	static bool getrusage_broken = FALSE;
 	static struct rusage prev_usage;
 	struct rusage usage;
 
 	memset(stats_r, 0, sizeof(*stats_r));
 	/* cputime */
-	if (getrusage(RUSAGE_SELF, &usage) < 0)
-		memset(&usage, 0, sizeof(usage));
-	if (timeval_diff_usecs(&usage.ru_stime, &prev_usage.ru_stime) < 0) {
+	if (getrusage(RUSAGE_SELF, &usage) < 0) {
+		if (!getrusage_broken) {
+			i_error("getrusage() failed: %m");
+			getrusage_broken = TRUE;
+		}
+		usage = prev_usage;
+	} else if (timeval_diff_usecs(&usage.ru_stime, &prev_usage.ru_stime) < 0) {
 		/* This seems to be a Linux bug. */
 		usage.ru_stime = prev_usage.ru_stime;
 	}

From 9f47b8c69789e8e61b1581e3fb1864a125bf276f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 6 May 2016 22:24:20 +0300
Subject: [PATCH 122/450] imap: Avoid assert-crash after a failed mailbox sync.

We didn't close the mailbox and we didn't update message counts, so we could
have crashed with:

Panic: Message count decreased
---
 src/imap/imap-sync.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/imap/imap-sync.c b/src/imap/imap-sync.c
index 5403605cf99..50ef1f3048a 100644
--- a/src/imap/imap-sync.c
+++ b/src/imap/imap-sync.c
@@ -315,7 +315,7 @@ static int imap_sync_finish(struct imap_sync_context *ctx, bool aborting)
 	if (mailbox_sync_deinit(&ctx->sync_ctx, &ctx->sync_status) < 0 ||
 	    ctx->failed) {
 		ctx->failed = TRUE;
-		return -1;
+		ret = -1;
 	}
 	mailbox_get_open_status(ctx->box, STATUS_UIDVALIDITY |
 				STATUS_MESSAGES | STATUS_RECENT |

From 8dfa9cf75a5f0811053a8d5e8170c2ce1421c291 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 6 May 2016 17:35:10 +0300
Subject: [PATCH 123/450] dict-client: Added idle_msecs=<n> parameter

This can be useful when connected to dict-async server where extra idling
connections don't matter so much as with blocking dict servers.
---
 src/lib-dict/dict-client.c | 28 +++++++++++++++++++++-------
 1 file changed, 21 insertions(+), 7 deletions(-)

diff --git a/src/lib-dict/dict-client.c b/src/lib-dict/dict-client.c
index 5eed547a4de..e4a4a4435a5 100644
--- a/src/lib-dict/dict-client.c
+++ b/src/lib-dict/dict-client.c
@@ -18,7 +18,7 @@
    handle only one client at a time. This is why the default timeout is zero,
    so that there won't be many dict processes just doing nothing. Zero means
    that the socket is disconnected immediately after returning to ioloop. */
-#define DICT_CLIENT_TIMEOUT_MSECS 0
+#define DICT_CLIENT_DEFAULT_TIMEOUT_MSECS 0
 
 /* Abort dict lookup after this many seconds. */
 #define DICT_CLIENT_READ_TIMEOUT_SECS 30
@@ -40,6 +40,7 @@ struct client_dict {
 	struct ostream *output;
 	struct io *io;
 	struct timeout *to_idle;
+	unsigned int idle_msecs;
 
 	struct client_dict_transaction_context *transactions;
 
@@ -409,11 +410,10 @@ static void client_dict_timeout(struct client_dict *dict)
 static void client_dict_add_timeout(struct client_dict *dict)
 {
 	if (dict->to_idle != NULL) {
-#if DICT_CLIENT_TIMEOUT_MSECS > 0
-		timeout_reset(dict->to_idle);
-#endif
+		if (dict->idle_msecs > 0)
+			timeout_reset(dict->to_idle);
 	} else if (client_dict_is_finished(dict)) {
-		dict->to_idle = timeout_add(DICT_CLIENT_TIMEOUT_MSECS,
+		dict->to_idle = timeout_add(dict->idle_msecs,
 					    client_dict_timeout, dict);
 	}
 }
@@ -509,10 +509,23 @@ client_dict_init(struct dict *driver, const char *uri,
 		 struct dict **dict_r, const char **error_r)
 {
 	struct client_dict *dict;
-	const char *dest_uri;
+	const char *p, *dest_uri;
+	unsigned int idle_msecs = DICT_CLIENT_DEFAULT_TIMEOUT_MSECS;
 	pool_t pool;
 
-	/* uri = [<path>] ":" <uri> */
+	/* uri = [idle_msecs=<n>:] [<path>] ":" <uri> */
+	if (strncmp(uri, "idle_msecs=", 11) == 0) {
+		p = strchr(uri+14, ':');
+		if (p == NULL) {
+			*error_r = t_strdup_printf("Invalid URI: %s", uri);
+			return -1;
+		}
+		if (str_to_uint(t_strdup_until(uri+14, p), &idle_msecs) < 0) {
+			*error_r = "Invalid idle_msecs";
+			return -1;
+		}
+		uri = p+1;
+	}
 	dest_uri = strchr(uri, ':');
 	if (dest_uri == NULL) {
 		*error_r = t_strdup_printf("Invalid URI: %s", uri);
@@ -525,6 +538,7 @@ client_dict_init(struct dict *driver, const char *uri,
 	dict->dict = *driver;
 	dict->value_type = set->value_type;
 	dict->username = p_strdup(pool, set->username);
+	dict->idle_msecs = idle_msecs;
 
 	dict->fd = -1;
 

From c2a17c802d691f5fc7eeec493613ce6c6965fc6e Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 6 May 2016 00:17:31 +0300
Subject: [PATCH 124/450] lib-http: If connect fails, include attempt count and
 total time in error.

This will produce errors such as:
9002 connect(1.2.3.4:801) failed: Connection timed out in 1.001 secs (4 attempts in 4.706 secs)
---
 src/lib-http/http-client-private.h |  1 +
 src/lib-http/http-client-queue.c   | 10 +++++++++-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/lib-http/http-client-private.h b/src/lib-http/http-client-private.h
index 27ad2d1fc4b..a5c0db766f2 100644
--- a/src/lib-http/http-client-private.h
+++ b/src/lib-http/http-client-private.h
@@ -208,6 +208,7 @@ struct http_client_queue {
 	   connected IP */
 	unsigned int ips_connect_start_idx;
 
+	struct timeval first_connect_time;
 	unsigned int connect_attempts;
 
 	/* peers we are trying to connect to;
diff --git a/src/lib-http/http-client-queue.c b/src/lib-http/http-client-queue.c
index 7b2bbff90b2..d6dea62ae04 100644
--- a/src/lib-http/http-client-queue.c
+++ b/src/lib-http/http-client-queue.c
@@ -282,7 +282,8 @@ void http_client_queue_connection_setup(struct http_client_queue *queue)
 				http_client_peer_addr2str(addr), ssl);
 
 			array_append(&queue->pending_peers, &peer, 1);
-			queue->connect_attempts++;
+			if (queue->connect_attempts++ == 0)
+				queue->first_connect_time = ioloop_timeval;
 		}
 
 		/* start soft connect time-out (but only if we have another IP left) */
@@ -401,6 +402,13 @@ http_client_queue_connection_failure(struct http_client_queue *queue,
 			queue->connect_attempts >= set->max_connect_attempts) {
 			http_client_queue_debug(queue,
 				"Failed to set up any connection; failing all queued requests");
+			if (queue->connect_attempts > 1) {
+				unsigned int total_msecs =
+					timeval_diff_msecs(&ioloop_timeval, &queue->first_connect_time);
+				reason = t_strdup_printf("%s (%u attempts in %u.%03u secs)",
+					reason, queue->connect_attempts,
+					total_msecs/1000, total_msecs%1000);
+			}
 			queue->connect_attempts = 0;
 			http_client_queue_fail(queue,
 				HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, reason);

From 5f59c92c2a9047419bdd29856d6a91e3bdcead83 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 9 May 2016 08:04:39 +0300
Subject: [PATCH 125/450] README: Added missing supported RFCs

---
 README | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/README b/README
index 109ba8f7533..32fe185f16b 100644
--- a/README
+++ b/README
@@ -63,13 +63,18 @@ IMAP extensions:
  5256       - IMAP SORT and THREAD Extensions
  5258       - IMAP4 - LIST Command Extensions 
  5267       - Contexts for IMAP4
+ 5464       - The IMAP METADATA Extension
+ 5465       - The IMAP NOTIFY Extension
  5524       - Extended URLFETCH for Binary and Converted Parts
  5530       - IMAP Response Codes
  5819       - IMAP4 Extension for Returning STATUS Information in Extended LIST
  5957       - Display-Based Address Sorting for the IMAP4 SORT Extension
  6154       - IMAP LIST Extension for Special-Use Mailboxes
  6203       - IMAP4 Extension for Fuzzy Search
+ 6785       - Support for IMAP Events in Sieve (via Pigeonhole plugin)
  6851       - Internet Message Access Protocol (IMAP) - MOVE Extension
+ 7162       - IMAP Extensions: Quick Flag Changes Resynchronization (CONDSTORE)
+              and Quick Mailbox Resynchronization (QRESYNC)
 
 Contact info
 ------------

From 99ce322e586996354583230b277ff87e6073cad5 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@rename-it.nl>
Date: Tue, 10 May 2016 00:42:51 +0200
Subject: [PATCH 126/450] Added liblzma bug to Valgrind suppressions file.

It caused `make test` to fail with valgrind.
Occurred at least for Debian Jessie with liblzma package version 5.1.1alpha+20120614-2+b3.
---
 run-test-valgrind.supp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/run-test-valgrind.supp b/run-test-valgrind.supp
index 03585511d8c..25aa3a2513f 100644
--- a/run-test-valgrind.supp
+++ b/run-test-valgrind.supp
@@ -4,3 +4,14 @@
    fun:malloc
    obj:*/bash
 }
+{
+   <liblzma>
+   Memcheck:Cond
+   obj:/lib/x86_64-linux-gnu/liblzma.so.5.0.0
+   obj:/lib/x86_64-linux-gnu/liblzma.so.5.0.0
+   obj:/lib/x86_64-linux-gnu/liblzma.so.5.0.0
+   obj:/lib/x86_64-linux-gnu/liblzma.so.5.0.0
+   obj:/lib/x86_64-linux-gnu/liblzma.so.5.0.0
+   fun:lzma_stream_encoder
+   fun:lzma_easy_encoder
+}

From 7c2d7b9e7a1611658e61d91de4341dc712c2d1ec Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 10 May 2016 10:42:33 -0400
Subject: [PATCH 127/450] lib-index: "first saved UID of the day" wasn't
 updated right always.

It was using the UID of the first append. But it may not have been the
lowest UID. Do this after the appends are first sorted by their UID.
---
 src/lib-index/mail-index-transaction-finish.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-index/mail-index-transaction-finish.c b/src/lib-index/mail-index-transaction-finish.c
index d399c33b9ab..dfd4da3c124 100644
--- a/src/lib-index/mail-index-transaction-finish.c
+++ b/src/lib-index/mail-index-transaction-finish.c
@@ -326,8 +326,8 @@ mail_index_transaction_convert_to_uids(struct mail_index_transaction *t)
 void mail_index_transaction_finish(struct mail_index_transaction *t)
 {
 	if (array_is_created(&t->appends)) {
-		mail_index_update_day_headers(t);
 		mail_index_transaction_sort_appends(t);
+		mail_index_update_day_headers(t);
 	}
 	mail_index_transaction_finish_flag_updates(t);
 

From bd0bf1233c56f448690df5ba495f18d7a0b15f1a Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 10 May 2016 16:22:23 -0400
Subject: [PATCH 128/450] lib: Fixed crash when closing a failed istream-concat

i_stream_concat_seek() could have set cur_input==NULL and we still
attempted to seek the cur_input at close time.
---
 src/lib/istream-concat.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/lib/istream-concat.c b/src/lib/istream-concat.c
index f490ec31717..fb9c20188ec 100644
--- a/src/lib/istream-concat.c
+++ b/src/lib/istream-concat.c
@@ -23,7 +23,10 @@ static void i_stream_concat_close(struct iostream_private *stream,
 	struct concat_istream *cstream = (struct concat_istream *)stream;
 	unsigned int i;
 
-	(void)i_stream_concat_skip(cstream);
+	if (cstream->istream.istream.stream_errno == 0) {
+		/* get the parent streams to the wanted offset */
+		(void)i_stream_concat_skip(cstream);
+	}
 
 	if (close_parent) {
 		for (i = 0; cstream->input[i] != NULL; i++)

From 1b228876c020f2be8323253f862cbdabd482f756 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 10 May 2016 17:25:20 -0400
Subject: [PATCH 129/450] lib-ldap: Add libdovecot dependency to
 libdovecot-ldap

Required for linking to work on OSX.
---
 src/lib-ldap/Makefile.am | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib-ldap/Makefile.am b/src/lib-ldap/Makefile.am
index b990e93fdf3..8fba6ae82ab 100644
--- a/src/lib-ldap/Makefile.am
+++ b/src/lib-ldap/Makefile.am
@@ -17,9 +17,9 @@ libdovecot_ldap_la_SOURCES = \
 	ldap-compare.c \
 	ldap-entry.c
 
-libdovecot_ldap_la_DEPENDENCIES =
+libdovecot_ldap_la_DEPENDENCIES = ../lib-dovecot/libdovecot.la
 libdovecot_ldap_la_LDFLAGS = -export-dynamic
-libdovecot_ldap_la_LIBADD = $(LDAP_LIBS)
+libdovecot_ldap_la_LIBADD = ../lib-dovecot/libdovecot.la $(LDAP_LIBS)
 
 headers = \
 	ldap-client.h

From ebf2e93ed33cf3206075127818456fddfebf7281 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 10 May 2016 17:35:43 -0400
Subject: [PATCH 130/450] lib-ldap: Reverted previous commit for now.

We don't have libdovecot built at at this stage. But we can't build it
earlier, because it needs lib-dict-extra, which needs lib-ldap. So we have
cyclical dependencies now.
---
 src/lib-ldap/Makefile.am | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib-ldap/Makefile.am b/src/lib-ldap/Makefile.am
index 8fba6ae82ab..b990e93fdf3 100644
--- a/src/lib-ldap/Makefile.am
+++ b/src/lib-ldap/Makefile.am
@@ -17,9 +17,9 @@ libdovecot_ldap_la_SOURCES = \
 	ldap-compare.c \
 	ldap-entry.c
 
-libdovecot_ldap_la_DEPENDENCIES = ../lib-dovecot/libdovecot.la
+libdovecot_ldap_la_DEPENDENCIES =
 libdovecot_ldap_la_LDFLAGS = -export-dynamic
-libdovecot_ldap_la_LIBADD = ../lib-dovecot/libdovecot.la $(LDAP_LIBS)
+libdovecot_ldap_la_LIBADD = $(LDAP_LIBS)
 
 headers = \
 	ldap-client.h

From a80011b6e3df30a7015959b06e29fa4e8e521305 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 12 May 2016 07:15:41 -0400
Subject: [PATCH 131/450] fs-randomfail: Set fs error always when injecting
 failure.

It was done in some places, but not everywhere.
---
 src/lib-fs/fs-randomfail.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/lib-fs/fs-randomfail.c b/src/lib-fs/fs-randomfail.c
index 62999204b1d..c3132a53b0f 100644
--- a/src/lib-fs/fs-randomfail.c
+++ b/src/lib-fs/fs-randomfail.c
@@ -260,7 +260,11 @@ static bool fs_random_fail(struct fs *_fs, enum fs_op op)
 
 	if (fs->op_probability[op] == 0)
 		return FALSE;
-	return (unsigned int)(rand() % 100) <= fs->op_probability[op];
+	if ((unsigned int)(rand() % 100) <= fs->op_probability[op]) {
+		fs_set_error(_fs, RANDOMFAIL_ERROR);
+		return TRUE;
+	}
+	return FALSE;
 }
 
 static bool

From 69ced82926a142af47034178a77de804037bdf72 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 12 May 2016 07:18:49 -0400
Subject: [PATCH 132/450] fs-metawrap: Propagate fs_write_stream_abort() to
 parent always

It needed to be done also when temp_output==NULL, because we had already
started sending it to parent, but async parent wasn't being finished.
---
 src/lib-fs/fs-metawrap.c | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/lib-fs/fs-metawrap.c b/src/lib-fs/fs-metawrap.c
index 9785ee7f43c..5fcb49e12f4 100644
--- a/src/lib-fs/fs-metawrap.c
+++ b/src/lib-fs/fs-metawrap.c
@@ -394,10 +394,17 @@ static int fs_metawrap_write_stream_finish(struct fs_file *_file, bool success)
 			o_stream_unref(&_file->output);
 	}
 	if (!success) {
-		if (file->temp_output != NULL)
-			o_stream_destroy(&file->temp_output);
-		if (file->super_output != NULL)
+		if (file->super_output != NULL) {
+			/* no metawrap */
+			i_assert(file->temp_output == NULL);
+			fs_write_stream_abort(file->super, &file->super_output);
+		} else if (file->temp_output == NULL) {
+			/* finishing up */
+			i_assert(file->super_output == NULL);
 			fs_write_stream_abort(file->super, &file->super_output);
+		} else {
+			o_stream_destroy(&file->temp_output);
+		}
 		return -1;
 	}
 

From 4c4a583c7339bce451b598ad19631df2d79c724d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 12 May 2016 07:21:38 -0400
Subject: [PATCH 133/450] fs-metawrap: Minor code cleanup

temp_output==NULL, so these calls are equal.
---
 src/lib-fs/fs-metawrap.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-fs/fs-metawrap.c b/src/lib-fs/fs-metawrap.c
index 5fcb49e12f4..6a4fd759c72 100644
--- a/src/lib-fs/fs-metawrap.c
+++ b/src/lib-fs/fs-metawrap.c
@@ -416,7 +416,7 @@ static int fs_metawrap_write_stream_finish(struct fs_file *_file, bool success)
 	if (file->temp_output == NULL) {
 		/* finishing up */
 		i_assert(file->super_output == NULL);
-		return fs_write_stream_finish(file->super, &file->temp_output);
+		return fs_write_stream_finish_async(file->super);
 	}
 	/* finish writing the temporary file */
 	input = iostream_temp_finish(&file->temp_output, IO_BLOCK_SIZE);

From 7bea15d9a4208c31a4b93764d39453f6e8d05565 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 12 May 2016 07:23:04 -0400
Subject: [PATCH 134/450] lib-fs: Added asserts to make sure async writes are
 finished before close

---
 src/lib-fs/fs-api-private.h |  1 +
 src/lib-fs/fs-api.c         | 13 +++++++++++++
 2 files changed, 14 insertions(+)

diff --git a/src/lib-fs/fs-api-private.h b/src/lib-fs/fs-api-private.h
index 6f931c19c29..e37104e8a68 100644
--- a/src/lib-fs/fs-api-private.h
+++ b/src/lib-fs/fs-api-private.h
@@ -115,6 +115,7 @@ struct fs_file {
 	struct timeval timing_start[FS_OP_COUNT];
 
 	unsigned int write_pending:1;
+	unsigned int writing_stream:1;
 	unsigned int metadata_changed:1;
 
 	unsigned int read_or_prefetch_counted:1;
diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c
index 2655b33d4c9..2acff9613b7 100644
--- a/src/lib-fs/fs-api.c
+++ b/src/lib-fs/fs-api.c
@@ -262,6 +262,9 @@ void fs_file_deinit(struct fs_file **_file)
 
 void fs_file_close(struct fs_file *file)
 {
+	i_assert(!file->writing_stream);
+	i_assert(file->output == NULL);
+
 	if (file->pending_read_input != NULL)
 		i_stream_unref(&file->pending_read_input);
 	if (file->seekable_input != NULL)
@@ -616,6 +619,10 @@ int fs_write(struct fs_file *file, const void *data, size_t size)
 
 struct ostream *fs_write_stream(struct fs_file *file)
 {
+	i_assert(!file->writing_stream);
+	i_assert(file->output == NULL);
+
+	file->writing_stream = TRUE;
 	file->fs->stats.write_count++;
 	T_BEGIN {
 		file->fs->v.write_stream(file);
@@ -629,6 +636,8 @@ static int fs_write_stream_finish_int(struct fs_file *file, bool success)
 {
 	int ret;
 
+	i_assert(file->writing_stream);
+
 	fs_file_timing_start(file, FS_OP_WRITE);
 	T_BEGIN {
 		ret = file->fs->v.write_stream_finish(file, success);
@@ -641,6 +650,8 @@ static int fs_write_stream_finish_int(struct fs_file *file, bool success)
 		   indicated a failure. */
 		i_assert(success);
 	}
+	if (ret != 0)
+		file->writing_stream = FALSE;
 	return ret;
 }
 
@@ -892,6 +903,8 @@ int fs_delete(struct fs_file *file)
 {
 	int ret;
 
+	i_assert(!file->writing_stream);
+
 	fs_file_timing_start(file, FS_OP_DELETE);
 	T_BEGIN {
 		ret = file->fs->v.delete_file(file);

From fece801a15b50e232d9b2cb11fbe507d327287a9 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 12 May 2016 07:23:51 -0400
Subject: [PATCH 135/450] lib-fs: Set fs error on fs_write_stream_abort()

Probably would be nice for fs_write_stream_abort() to have an error string
parameter, which could be used instead of this generic error.
---
 src/lib-fs/fs-api.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c
index 2acff9613b7..443e5a35563 100644
--- a/src/lib-fs/fs-api.c
+++ b/src/lib-fs/fs-api.c
@@ -687,6 +687,8 @@ void fs_write_stream_abort(struct fs_file *file, struct ostream **output)
 	*output = NULL;
 	if (file->output != NULL)
 		o_stream_ignore_last_errors(file->output);
+	/* make sure we don't have an old error lying around */
+	fs_set_error(file->fs, "Write aborted");
 	(void)fs_write_stream_finish_int(file, FALSE);
 }
 

From 5aa642046277a4dbbbbc4fcf3de750d5c8760c18 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 12 May 2016 07:24:39 -0400
Subject: [PATCH 136/450] lib-fs: Minor code cleanup

---
 src/lib-fs/fs-api.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c
index 443e5a35563..ba2d09324ea 100644
--- a/src/lib-fs/fs-api.c
+++ b/src/lib-fs/fs-api.c
@@ -662,9 +662,8 @@ int fs_write_stream_finish(struct fs_file *file, struct ostream **output)
 	i_assert(*output == file->output || *output == NULL);
 
 	*output = NULL;
-	if (file->output != NULL)
-		o_stream_uncork(file->output);
 	if (file->output != NULL) {
+		o_stream_uncork(file->output);
 		if (o_stream_nfinish(file->output) < 0) {
 			fs_set_error(file->fs, "write(%s) failed: %s",
 				     o_stream_get_name(file->output),

From fa4f8f36b213d233e25f628f48a8f3b33d842b88 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 11 May 2016 16:58:59 -0400
Subject: [PATCH 137/450] dict-sql: Cache reading settings files.

The settings were read for every dict init, which was done for every new
dict connection. This was using a lot of CPU. There are usually only a
couple dict-sql settings files, so we cache all of the ones we read.
---
 src/lib-dict/dict-sql-settings.c | 49 ++++++++++++++++++++++++++++++--
 src/lib-dict/dict-sql-settings.h |  4 ++-
 src/lib-dict/dict-sql.c          |  3 +-
 3 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/src/lib-dict/dict-sql-settings.c b/src/lib-dict/dict-sql-settings.c
index 8e5614633f8..5b090061edd 100644
--- a/src/lib-dict/dict-sql-settings.c
+++ b/src/lib-dict/dict-sql-settings.c
@@ -3,6 +3,7 @@
 #include "lib.h"
 #include "array.h"
 #include "str.h"
+#include "hash.h"
 #include "settings.h"
 #include "dict-sql-settings.h"
 
@@ -42,6 +43,14 @@ static const struct setting_def dict_sql_map_setting_defs[] = {
 	{ 0, NULL, 0 }
 };
 
+struct dict_sql_settings_cache {
+	pool_t pool;
+	const char *path;
+	struct dict_sql_settings *set;
+};
+
+static HASH_TABLE(const char *, struct dict_sql_settings_cache *) dict_sql_settings_cache;
+
 static const char *pattern_read_name(const char **pattern)
 {
 	const char *p = *pattern, *name;
@@ -235,9 +244,20 @@ parse_section(const char *type, const char *name ATTR_UNUSED,
 }
 
 struct dict_sql_settings *
-dict_sql_settings_read(pool_t pool, const char *path, const char **error_r)
+dict_sql_settings_read(const char *path, const char **error_r)
 {
 	struct setting_parser_ctx ctx;
+	struct dict_sql_settings_cache *cache;
+	pool_t pool = pool_alloconly_create("dict sql settings", 1024);
+
+	if (!hash_table_is_created(dict_sql_settings_cache)) {
+		hash_table_create(&dict_sql_settings_cache, default_pool, 0,
+				  str_hash, strcmp);
+	}
+
+	cache = hash_table_lookup(dict_sql_settings_cache, path);
+	if (cache != NULL)
+		return cache->set;
 
 	memset(&ctx, 0, sizeof(ctx));
 	ctx.pool = pool;
@@ -246,14 +266,39 @@ dict_sql_settings_read(pool_t pool, const char *path, const char **error_r)
 	p_array_init(&ctx.set->maps, pool, 8);
 
 	if (!settings_read(path, NULL, parse_setting, parse_section,
-			   &ctx, error_r))
+			   &ctx, error_r)) {
+		pool_unref(&pool);
 		return NULL;
+	}
 
 	if (ctx.set->connect == NULL) {
 		*error_r = t_strdup_printf("Error in configuration file %s: "
 					   "Missing connect setting", path);
+		pool_unref(&pool);
 		return NULL;
 	}
 
+	cache = p_new(pool, struct dict_sql_settings_cache, 1);
+	cache->pool = pool;
+	cache->path = p_strdup(pool, path);
+	cache->set = ctx.set;
+
+	hash_table_insert(dict_sql_settings_cache, cache->path, cache);
 	return ctx.set;
 }
+
+void dict_sql_settings_deinit(void)
+{
+	struct hash_iterate_context *iter;
+	struct dict_sql_settings_cache *cache;
+	const char *key;
+
+	if (!hash_table_is_created(dict_sql_settings_cache))
+		return;
+
+	iter = hash_table_iterate_init(dict_sql_settings_cache);
+	while (hash_table_iterate(iter, dict_sql_settings_cache, &key, &cache))
+		pool_unref(&cache->pool);
+	hash_table_iterate_deinit(&iter);
+	hash_table_destroy(&dict_sql_settings_cache);
+}
diff --git a/src/lib-dict/dict-sql-settings.h b/src/lib-dict/dict-sql-settings.h
index 62aff2ec17f..dd9537237bb 100644
--- a/src/lib-dict/dict-sql-settings.h
+++ b/src/lib-dict/dict-sql-settings.h
@@ -33,6 +33,8 @@ struct dict_sql_settings {
 };
 
 struct dict_sql_settings *
-dict_sql_settings_read(pool_t pool, const char *path, const char **error_r);
+dict_sql_settings_read(const char *path, const char **error_r);
+
+void dict_sql_settings_deinit(void);
 
 #endif
diff --git a/src/lib-dict/dict-sql.c b/src/lib-dict/dict-sql.c
index 6af83ece99b..4a0a140ae66 100644
--- a/src/lib-dict/dict-sql.c
+++ b/src/lib-dict/dict-sql.c
@@ -88,7 +88,7 @@ sql_dict_init(struct dict *driver, const char *uri,
 	dict->pool = pool;
 	dict->dict = *driver;
 	dict->username = p_strdup(pool, set->username);
-	dict->set = dict_sql_settings_read(pool, uri, error_r);
+	dict->set = dict_sql_settings_read(uri, error_r);
 	if (dict->set == NULL) {
 		pool_unref(&pool);
 		return -1;
@@ -1289,4 +1289,5 @@ void dict_sql_unregister(void)
 		dict_driver_unregister(&dict_sql_drivers[i]);
 	i_free(dict_sql_drivers);
 	sql_db_cache_deinit(&dict_sql_db_cache);
+	dict_sql_settings_deinit();
 }

From 9f300ec1af1f711ab7abe350eb333d7487384a25 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 12 May 2016 08:09:56 -0400
Subject: [PATCH 138/450] lib-http: Queue's delayed timeout handler wasn't
 removed after work was done.

---
 src/lib-http/http-client-queue.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lib-http/http-client-queue.c b/src/lib-http/http-client-queue.c
index d6dea62ae04..42f24e11600 100644
--- a/src/lib-http/http-client-queue.c
+++ b/src/lib-http/http-client-queue.c
@@ -639,6 +639,7 @@ http_client_queue_delay_timeout(struct http_client_queue *queue)
 	struct http_client_request *const *reqs;
 	unsigned int count, i, finished;
 
+	timeout_remove(&queue->to_delayed);
 	io_loop_time_refresh();
 
 	finished = 0;

From 826f16c0cd4c1ea667a441f45caf48e913bac906 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 28 Apr 2016 12:47:02 +0300
Subject: [PATCH 139/450] lib-fs: Fixed multiple concurrent fs_read_stream()
 calls

Return a new limit-istream, so each istream can have its own independent
offset.
---
 src/lib-fs/fs-api.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c
index ba2d09324ea..5247636be6e 100644
--- a/src/lib-fs/fs-api.c
+++ b/src/lib-fs/fs-api.c
@@ -512,9 +512,10 @@ struct istream *fs_read_stream(struct fs_file *file, size_t max_buffer_size)
 	}
 
 	if (file->seekable_input != NULL) {
-		i_stream_seek(file->seekable_input, 0);
-		i_stream_ref(file->seekable_input);
-		return file->seekable_input;
+		/* allow multiple open streams, each in a different position */
+		input = i_stream_create_limit(file->seekable_input, (uoff_t)-1);
+		i_stream_seek(input, 0);
+		return input;
 	}
 	T_BEGIN {
 		input = file->fs->v.read_stream(file, max_buffer_size);

From d375835dd9fdff87c051aa646e54a1f8693fe7bc Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 28 Apr 2016 12:45:01 +0300
Subject: [PATCH 140/450] lib-fs: Allow multiple fs_read_stream() calls to
 seekable istreams.

This fixes for example calling fs_default_copy() for an istream which was
already opened:

Panic: file fs-api.c: line 528 (fs_read_stream): assertion failed: (!file->istream_open)
---
 src/lib-fs/fs-api.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c
index 5247636be6e..f57774676f6 100644
--- a/src/lib-fs/fs-api.c
+++ b/src/lib-fs/fs-api.c
@@ -545,10 +545,10 @@ struct istream *fs_read_stream(struct fs_file *file, size_t max_buffer_size)
 						file->fs->temp_path_prefix);
 		i_stream_set_name(input, i_stream_get_name(inputs[0]));
 		i_stream_unref(&inputs[0]);
-
-		file->seekable_input = input;
-		i_stream_ref(file->seekable_input);
 	}
+	file->seekable_input = input;
+	i_stream_ref(file->seekable_input);
+
 	if ((file->flags & FS_OPEN_FLAG_ASYNC) == 0 && !input->blocking) {
 		/* read the whole input stream before returning */
 		while ((ret = i_stream_read_data(input, &data, &size, 0)) >= 0) {

From f5f2a7a7f2259db3ba42964fbbaeadb8870e64bf Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 12 May 2016 15:10:47 -0400
Subject: [PATCH 141/450] lib-fs: Added FS_PROPERTY_FASTCOPY_CHANGED_METADATA

---
 src/lib-fs/fs-api.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/lib-fs/fs-api.h b/src/lib-fs/fs-api.h
index c21080386c2..486e7fdf176 100644
--- a/src/lib-fs/fs-api.h
+++ b/src/lib-fs/fs-api.h
@@ -36,7 +36,9 @@ enum fs_properties {
 	/* Backend support asynchronous file operations. */
 	FS_PROPERTY_ASYNC		= 0x800,
 	/* Backend supports FS_ITER_FLAG_OBJECTIDS. */
-	FS_PROPERTY_OBJECTIDS		= 0x1000
+	FS_PROPERTY_OBJECTIDS		= 0x1000,
+	/* fs_copy() is fast even when file's metadata is changed */
+	FS_PROPERTY_FASTCOPY_CHANGED_METADATA = 0x2000,
 };
 
 enum fs_open_mode {

From 6bc001ee9dc03cb3107239861867cd674fd321d7 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 13 May 2016 09:23:52 -0400
Subject: [PATCH 142/450] lib: Fixed potential crash in i_stream_stat()
 failures.

We shouldn't have been copying parent's stream_errno here. Especially
because the parent can be NULL.
---
 src/lib/istream.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/lib/istream.c b/src/lib/istream.c
index d525c8b7701..c5bf2d6c29e 100644
--- a/src/lib/istream.c
+++ b/src/lib/istream.c
@@ -808,10 +808,8 @@ static int
 i_stream_default_get_size(struct istream_private *stream,
 			  bool exact, uoff_t *size_r)
 {
-	if (stream->stat(stream, exact) < 0) {
-		stream->istream.stream_errno = stream->parent->stream_errno;
+	if (stream->stat(stream, exact) < 0)
 		return -1;
-	}
 	if (stream->statbuf.st_size == -1)
 		return 0;
 

From 4132ed5a3b2350d78188bb184d98ee6659ee5e1d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 13 May 2016 09:48:13 -0400
Subject: [PATCH 143/450] lib-fs: If fs-metawrap sees truncated header, it
 should return error.

---
 src/lib-fs/istream-metawrap.c | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/lib-fs/istream-metawrap.c b/src/lib-fs/istream-metawrap.c
index c389432d7df..c869736278c 100644
--- a/src/lib-fs/istream-metawrap.c
+++ b/src/lib-fs/istream-metawrap.c
@@ -33,8 +33,15 @@ static int metadata_header_read(struct metawrap_istream *mstream)
 		mstream->callback(line, p, mstream->context);
 	}
 	if (mstream->istream.parent->eof) {
-		mstream->istream.istream.stream_errno =
-			mstream->istream.parent->stream_errno;
+		if (mstream->istream.parent->stream_errno != 0) {
+			mstream->istream.istream.stream_errno =
+				mstream->istream.parent->stream_errno;
+		} else {
+			io_stream_set_error(&mstream->istream.iostream,
+				"Metadata header is missing ending line");
+			mstream->istream.istream.stream_errno = EINVAL;
+			return -1;
+		}
 		mstream->istream.istream.eof = TRUE;
 		return -1;
 	}

From aae505b6151745fdc60e46969b6ddf45d5b3f6be Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 13 May 2016 09:56:08 -0400
Subject: [PATCH 144/450] lib-fs: Make sure fs-metawrap catches all write
 errors.

The full istream may not have been written in case ostream only partially
wrote the data (e.g. out of disk space?)
---
 src/lib-fs/fs-metawrap.c | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/lib-fs/fs-metawrap.c b/src/lib-fs/fs-metawrap.c
index 6a4fd759c72..ceb2b84b41d 100644
--- a/src/lib-fs/fs-metawrap.c
+++ b/src/lib-fs/fs-metawrap.c
@@ -430,21 +430,22 @@ static int fs_metawrap_write_stream_finish(struct fs_file *_file, bool success)
 		i_stream_unref(&input2);
 	}
 	file->super_output = fs_write_stream(file->super);
-	if (o_stream_send_istream(file->super_output, input) >= 0)
-		ret = fs_write_stream_finish(file->super, &file->super_output);
-	else if (input->stream_errno != 0) {
+	(void)o_stream_send_istream(file->super_output, input);
+	if (input->stream_errno != 0) {
 		fs_set_error(_file->fs, "read(%s) failed: %s",
 			     i_stream_get_name(input),
 			     i_stream_get_error(input));
 		fs_write_stream_abort(file->super, &file->super_output);
 		ret = -1;
-	} else {
-		i_assert(file->super_output->stream_errno != 0);
+	} else if (file->super_output->stream_errno != 0) {
 		fs_set_error(_file->fs, "write(%s) failed: %s",
 			     o_stream_get_name(file->super_output),
 			     o_stream_get_error(file->super_output));
 		fs_write_stream_abort(file->super, &file->super_output);
 		ret = -1;
+	} else {
+		i_assert(i_stream_is_eof(input));
+		ret = fs_write_stream_finish(file->super, &file->super_output);
 	}
 	i_stream_unref(&input);
 	return ret;

From a0b127aa8a84e5106d8848bca0db0b09544290fb Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 13 May 2016 13:35:50 -0400
Subject: [PATCH 145/450] lib: Fixed istream_seekable.stat() to return correct
 size.

It was supposed to read until end of the stream, but it did it only when
stat() was called while stream was still fully in memory.
---
 src/lib/istream-seekable.c | 35 +++++++++++++++++------------------
 1 file changed, 17 insertions(+), 18 deletions(-)

diff --git a/src/lib/istream-seekable.c b/src/lib/istream-seekable.c
index a12581813c9..f6340be5f95 100644
--- a/src/lib/istream-seekable.c
+++ b/src/lib/istream-seekable.c
@@ -316,25 +316,24 @@ i_stream_seekable_stat(struct istream_private *stream, bool exact)
 		return 0;
 	}
 
-	if (sstream->membuf != NULL) {
-		/* we want to know the full size of the file, so read until
-		   we're finished */
-		old_offset = stream->istream.v_offset;
-		do {
-			i_stream_skip(&stream->istream,
-				      stream->pos - stream->skip);
-		} while ((ret = i_stream_seekable_read(stream)) > 0);
-
-		if (ret == 0) {
-			i_panic("i_stream_stat() used for non-blocking "
-				"seekable stream %s offset %"PRIuUOFF_T,
-				i_stream_get_name(sstream->cur_input),
-				sstream->cur_input->v_offset);
-		}
-		i_stream_skip(&stream->istream, stream->pos - stream->skip);
-		i_stream_seek(&stream->istream, old_offset);
-		unref_streams(sstream);
+	/* we want to know the full size of the file, so read until
+	   we're finished */
+	old_offset = stream->istream.v_offset;
+	do {
+		i_stream_skip(&stream->istream,
+			      stream->pos - stream->skip);
+	} while ((ret = i_stream_seekable_read(stream)) > 0);
+
+	if (ret == 0) {
+		i_panic("i_stream_stat() used for non-blocking "
+			"seekable stream %s offset %"PRIuUOFF_T,
+			i_stream_get_name(sstream->cur_input),
+			sstream->cur_input->v_offset);
 	}
+	i_stream_skip(&stream->istream, stream->pos - stream->skip);
+	i_stream_seek(&stream->istream, old_offset);
+	unref_streams(sstream);
+
 	if (stream->istream.stream_errno != 0)
 		return -1;
 

From 26758def5493a93eeb40414830917b73e7d84407 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 13 May 2016 14:22:40 -0400
Subject: [PATCH 146/450] imap: Set command start timestamps earlier.

Previously timing statistics in taglines weren't shown commands that didn't
read any parameters.

Also the timings now include the time speng reading command parameters from
client. For example:

a list "" {1}
+ OK
%
* LIST (\HasNoChildren) "/" INBOX
a OK List completed (0.001 + 1.214 secs).
---
 src/imap/imap-client.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/imap/imap-client.c b/src/imap/imap-client.c
index 76424f26b1a..6392d5ff76f 100644
--- a/src/imap/imap-client.c
+++ b/src/imap/imap-client.c
@@ -584,9 +584,6 @@ bool client_read_args(struct client_command_context *cmd, unsigned int count,
 		str = t_str_new(256);
 		imap_write_args(str, *args_r);
 		cmd->args = p_strdup(cmd->pool, str_c(str));
-		cmd->start_time = ioloop_timeval;
-		cmd->start_ioloop_wait_usecs =
-			io_loop_get_wait_usecs(current_ioloop);
 
 		cmd->client->input_lock = NULL;
 		return TRUE;
@@ -723,6 +720,8 @@ struct client_command_context *client_command_alloc(struct client *client)
 	cmd = p_new(client->command_pool, struct client_command_context, 1);
 	cmd->client = client;
 	cmd->pool = client->command_pool;
+	cmd->start_time = ioloop_timeval;
+	cmd->start_ioloop_wait_usecs = io_loop_get_wait_usecs(current_ioloop);
 	p_array_init(&cmd->module_contexts, cmd->pool, 5);
 
 	DLLIST_PREPEND(&client->command_queue, cmd);

From b35c9039d95af2753253f8fa5c052c3114e0b11d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 6 May 2016 13:42:11 +0300
Subject: [PATCH 147/450] dict-redis: Fixed memory leak in async commit

---
 src/lib-dict/dict-redis.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/lib-dict/dict-redis.c b/src/lib-dict/dict-redis.c
index 6aa38247f1f..8ff7160c3bb 100644
--- a/src/lib-dict/dict-redis.c
+++ b/src/lib-dict/dict-redis.c
@@ -623,8 +623,10 @@ redis_transaction_commit(struct dict_transaction_context *_ctx, bool async,
 		redis_input_state_add(dict, REDIS_INPUT_STATE_EXEC);
 		for (i = 0; i < ctx->cmd_count; i++)
 			redis_input_state_add(dict, REDIS_INPUT_STATE_EXEC_REPLY);
-		if (async)
+		if (async) {
+			i_free(ctx);
 			return 1;
+		}
 		redis_wait(dict);
 	}
 	if (callback != NULL)

From 71bcaeb23954e591413dd5661df61caa3db17262 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 11 May 2016 16:57:33 -0400
Subject: [PATCH 148/450] lib-sql: Memory leak fix when freeing sql dbs from
 cache

Normally happened only at deinit.
---
 src/lib-sql/sql-db-cache.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lib-sql/sql-db-cache.c b/src/lib-sql/sql-db-cache.c
index d035aaa2240..7b84f229528 100644
--- a/src/lib-sql/sql-db-cache.c
+++ b/src/lib-sql/sql-db-cache.c
@@ -79,6 +79,7 @@ static void sql_db_cache_free_tail(struct sql_db_cache *cache)
 
 	i_free(ctx->key);
 	ctx->orig_deinit(db);
+	i_free(ctx);
 }
 
 static void sql_db_cache_drop_oldest(struct sql_db_cache *cache)

From 015c15fd5fd64abfcb55ee93a54f0a5c32be4082 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 15 May 2016 01:17:18 +0300
Subject: [PATCH 149/450] lib-ldap: Compiling fix - don't pass through void
 return

---
 src/lib-ldap/ldap-client.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib-ldap/ldap-client.c b/src/lib-ldap/ldap-client.c
index bdafd87b115..bb679f7543c 100644
--- a/src/lib-ldap/ldap-client.c
+++ b/src/lib-ldap/ldap-client.c
@@ -55,7 +55,7 @@ void ldap_search_start(struct ldap_client *client,
 {
 	/* FIXME: we could support multiple concurrent LDAP connections to
 	   the same host. */
-	return ldap_connection_search_start(client->list->conn, input, callback, context);
+	ldap_connection_search_start(client->list->conn, input, callback, context);
 }
 
 #undef ldap_compare_start
@@ -63,7 +63,7 @@ void ldap_compare_start(struct ldap_client *client,
 			const struct ldap_compare_input *input,
 			ldap_result_callback_t *callback, void *context)
 {
-	return ldap_connection_compare_start(client->list->conn, input, callback, context);
+	ldap_connection_compare_start(client->list->conn, input, callback, context);
 }
 
 void ldap_clients_cleanup(void)

From 3a02e96bd6902de51711735bc282801ccd13c6a9 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Sun, 15 May 2016 17:25:17 +0300
Subject: [PATCH 150/450] lib-ldap: Do not create new context and lose settings

---
 src/lib-ldap/ldap-connection.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/lib-ldap/ldap-connection.c b/src/lib-ldap/ldap-connection.c
index e26ced40f73..35dd0d7c76e 100644
--- a/src/lib-ldap/ldap-connection.c
+++ b/src/lib-ldap/ldap-connection.c
@@ -88,6 +88,9 @@ int ldap_connection_setup(struct ldap_connection *conn, const char **error_r)
 
 	ldap_set_option(conn->conn, LDAP_OPT_REFERRALS, 0);
 
+	opt = 0;
+	ldap_set_option(conn->conn, LDAP_OPT_X_TLS_NEWCTX, &opt);
+
 	return 0;
 }
 

From 72c15660f6caaf0fab9682e290a50dc9e3b7097b Mon Sep 17 00:00:00 2001
From: Phil Carmody <phil@dovecot.fi>
Date: Mon, 8 Feb 2016 18:32:10 +0200
Subject: [PATCH 151/450] lib: istream - provide alternatives to
 i_stream_read_data()

Providing as a parameter the number of bytes that you would be
disappointed to receive was confusing compared to providing the
number that you would be happy to receive. This lets us get rid
of all the '-1's in the callers. The callers which used 0 as
a parameter were so common that it's worth providing a separate
API just to simplify their demands.

Deep down, these are still the same underlying function. Currently
we route the new API through the old one, but eventually, the old
API can be routed through the new one, so that the friendlier
interface has the simplest implementation.

Signed-off-by: Phil Carmody <phil@dovecot.fi>
---
 src/lib/istream.h | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/lib/istream.h b/src/lib/istream.h
index 025afd0f54c..4db812005d9 100644
--- a/src/lib/istream.h
+++ b/src/lib/istream.h
@@ -165,6 +165,24 @@ unsigned char *i_stream_get_modifiable_data(struct istream *stream,
    input buffer is full. */
 int i_stream_read_data(struct istream *stream, const unsigned char **data_r,
 		       size_t *size_r, size_t threshold);
+/* Like i_stream_get_data(), but read more when needed. Returns 1 if at least
+   the wanted number of bytes are available, 0 if less, -1 if error or
+   EOF with no bytes read that weren't already in buffer, or -2 if stream's
+   input buffer is full. */
+static inline int
+i_stream_read_bytes(struct istream *stream, const unsigned char **data_r,
+			size_t *size_r, size_t wanted)
+{
+	i_assert(wanted > 0);
+	return i_stream_read_data(stream, data_r, size_r, wanted - 1);
+}
+/* Short-hand for just requesting more data (i.e. even one byte) */
+static inline int
+i_stream_read_more(struct istream *stream, const unsigned char **data_r,
+		   size_t *size_r)
+{
+	return i_stream_read_bytes(stream, data_r, size_r, 1);
+}
 
 /* Append external data to input stream. Returns TRUE if successful, FALSE if
    there is not enough space in the stream. */

From b6d4252dc8c01b45055796ad290f3fd7cb5ff88a Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 12 May 2016 17:45:34 -0400
Subject: [PATCH 152/450] lib-ldap: Fixed assert-crash when killing LDAP
 connection.

libldap may have already closed the fd, so with epoll io_remove() would
try to remove it and crash with:

Panic: epoll_ctl(del, 12) failed: Bad file descriptor
---
 src/lib-ldap/ldap-connection.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-ldap/ldap-connection.c b/src/lib-ldap/ldap-connection.c
index 35dd0d7c76e..1322c9341d4 100644
--- a/src/lib-ldap/ldap-connection.c
+++ b/src/lib-ldap/ldap-connection.c
@@ -552,7 +552,7 @@ int ldap_connection_connect(struct ldap_connection *conn)
 void ldap_connection_kill(struct ldap_connection *conn)
 {
 	if (conn->io != NULL)
-		io_remove(&(conn->io));
+		io_remove_closed(&(conn->io));
 	if (conn->to_disconnect != NULL)
 		timeout_remove(&(conn->to_disconnect));
 	if (conn->to_reconnect != NULL)

From 9b496d96f9c985c29b398d2f4c33a7163239c52e Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Tue, 17 May 2016 23:31:29 +0300
Subject: [PATCH 153/450] doveadm-mailbox: Add update subcommand

---
 src/doveadm/doveadm-mail-mailbox.c | 113 +++++++++++++++++++++++++++++
 src/doveadm/doveadm-mail.c         |   1 +
 src/doveadm/doveadm-mail.h         |   1 +
 3 files changed, 115 insertions(+)

diff --git a/src/doveadm/doveadm-mail-mailbox.c b/src/doveadm/doveadm-mail-mailbox.c
index 7f16fed061d..2050b24b5bc 100644
--- a/src/doveadm/doveadm-mail-mailbox.c
+++ b/src/doveadm/doveadm-mail-mailbox.c
@@ -47,6 +47,12 @@ struct list_cmd_context {
 	bool mutf7;
 };
 
+struct update_cmd_context {
+	struct doveadm_mailbox_cmd_context ctx;
+	const char *mailbox;
+	struct mailbox_update update;
+};
+
 void doveadm_mailbox_args_check(const char *const args[])
 {
 	unsigned int i;
@@ -556,6 +562,97 @@ static struct doveadm_mail_cmd_context *cmd_mailbox_unsubscribe_alloc(void)
 	return cmd_mailbox_subscriptions_alloc(FALSE);
 }
 
+static
+void cmd_mailbox_update_init(struct doveadm_mail_cmd_context *_ctx,
+			     const char *const args[])
+{
+	struct update_cmd_context *ctx = (struct update_cmd_context *)_ctx;
+
+	if (str_array_length(args) != 1)
+		doveadm_mail_help_name("mailbox update");
+
+	doveadm_mailbox_args_check(args);
+
+	ctx->mailbox = args[0];
+
+	if ((ctx->update.min_first_recent_uid != 0 ||
+	     ctx->update.min_next_uid != 0) &&
+	    ctx->update.min_first_recent_uid > ctx->update.min_next_uid) {
+		i_fatal_status(EX_DATAERR,
+			"min_first_recent_uid > min_next_uid");
+	}
+}
+
+static
+bool cmd_mailbox_update_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+	struct update_cmd_context *ctx = (struct update_cmd_context *)_ctx;
+
+	switch (c) {
+	case 'g':
+		if (guid_128_from_string(optarg, ctx->update.mailbox_guid) < 0)
+			doveadm_mail_help_name("mailbox update");
+		break;
+	case 'V':
+		if (str_to_uint32(optarg, &(ctx->update.uid_validity)) < 0)
+			doveadm_mail_help_name("mailbox update");
+		break;
+	case 'N':
+		if (str_to_uint32(optarg, &(ctx->update.min_next_uid)) < 0)
+			doveadm_mail_help_name("mailbox update");
+		break;
+	case 'R':
+		if (str_to_uint32(optarg, &(ctx->update.min_first_recent_uid)) < 0)
+			doveadm_mail_help_name("mailbox update");
+		break;
+	case 'H':
+		if (str_to_uint64(optarg, &(ctx->update.min_highest_modseq)) < 0)
+			doveadm_mail_help_name("mailbox update");
+		break;
+	case 'P':
+		if (str_to_uint64(optarg, &(ctx->update.min_highest_pvt_modseq)) < 0)
+			doveadm_mail_help_name("mailbox update");
+		break;
+	default:
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static
+int cmd_mailbox_update_run(struct doveadm_mail_cmd_context *_ctx,
+			   struct mail_user *user)
+{
+	struct update_cmd_context *ctx = (struct update_cmd_context *)_ctx;
+	struct mail_namespace *ns;
+	struct mailbox *box;
+	int ret = 0;
+
+	ns = mail_namespace_find(user->namespaces, ctx->mailbox);
+	box = mailbox_alloc(ns->list, ctx->mailbox, 0);
+
+	if ((ret = mailbox_update(box, &(ctx->update))) != 0) {
+		i_error("Cannot update %s: %s",
+			ctx->mailbox,
+			mailbox_get_last_error(box, NULL));
+	}
+
+	mailbox_free(&box);
+
+	return ret;
+}
+
+static
+struct doveadm_mail_cmd_context *cmd_mailbox_update_alloc(void)
+{
+	struct update_cmd_context *ctx;
+	ctx = doveadm_mail_cmd_alloc(struct update_cmd_context);
+	ctx->ctx.ctx.v.parse_arg = cmd_mailbox_update_parse_arg;
+	ctx->ctx.ctx.v.init = cmd_mailbox_update_init;
+	ctx->ctx.ctx.v.run = cmd_mailbox_update_run;
+	return &ctx->ctx.ctx;
+}
+
 struct doveadm_cmd_ver2 doveadm_cmd_mailbox_list_ver2 = {
 	.name = "mailbox list",
 	.mail_cmd = cmd_mailbox_list_alloc,
@@ -623,3 +720,19 @@ DOVEADM_CMD_MAIL_COMMON
 DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
 DOVEADM_CMD_PARAMS_END
 };
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_update_ver2 = {
+	.name = "mailbox update",
+	.mail_cmd = cmd_mailbox_update_alloc,
+	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[--mailbox-guid guid] [--uid-validity uid] [--min-next-uid uid] [--min-first-recent-uid uid] [--min-highest-modseq seq] [--min-highest-pvt-modseq seq] <mailbox>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('g', "mailbox-guid", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('V', "uid-validity", CMD_PARAM_INT64, 0)
+DOVEADM_CMD_PARAM('N', "min-next-uid", CMD_PARAM_INT64, 0)
+DOVEADM_CMD_PARAM('R', "min-first-recent-uid", CMD_PARAM_INT64, 0)
+DOVEADM_CMD_PARAM('H', "min-highest-modseq", CMD_PARAM_INT64, 0)
+DOVEADM_CMD_PARAM('P', "min-highest-pvt-modseq", CMD_PARAM_INT64, 0)
+DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail.c b/src/doveadm/doveadm-mail.c
index 2b527d6a287..39faca2d98f 100644
--- a/src/doveadm/doveadm-mail.c
+++ b/src/doveadm/doveadm-mail.c
@@ -870,6 +870,7 @@ static struct doveadm_cmd_ver2 *mail_commands_ver2[] = {
 	&doveadm_cmd_mailbox_rename_ver2,
 	&doveadm_cmd_mailbox_subscribe_ver2,
 	&doveadm_cmd_mailbox_unsubscribe_ver2,
+	&doveadm_cmd_mailbox_update_ver2,
 	&doveadm_cmd_fetch_ver2,
 	&doveadm_cmd_save_ver2,
 	&doveadm_cmd_index_ver2,
diff --git a/src/doveadm/doveadm-mail.h b/src/doveadm/doveadm-mail.h
index b33543899fa..fce8204fdcd 100644
--- a/src/doveadm/doveadm-mail.h
+++ b/src/doveadm/doveadm-mail.h
@@ -197,6 +197,7 @@ extern struct doveadm_cmd_ver2 doveadm_cmd_import_ver2;
 extern struct doveadm_cmd_ver2 doveadm_cmd_search_ver2;
 extern struct doveadm_cmd_ver2 doveadm_cmd_copy_ver2;
 extern struct doveadm_cmd_ver2 doveadm_cmd_move_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_update_ver2;
 
 #define DOVEADM_CMD_MAIL_COMMON \
 DOVEADM_CMD_PARAM('A', "all-users", CMD_PARAM_BOOL, 0) \

From f008446315fdc50f4f8cdf5c786e885179afd333 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 17 May 2016 00:00:43 +0300
Subject: [PATCH 154/450] imap: Include sync timing information in tagged
 command replies. Show it only when it's larger than 0 to avoid unnecessary
 output.

---
 src/imap/imap-client.c | 11 ++++++++++-
 src/imap/imap-client.h |  3 +++
 src/imap/imap-sync.c   |  1 +
 3 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/src/imap/imap-client.c b/src/imap/imap-client.c
index 6392d5ff76f..b5046015643 100644
--- a/src/imap/imap-client.c
+++ b/src/imap/imap-client.c
@@ -480,6 +480,7 @@ client_cmd_append_timing_stats(struct client_command_context *cmd,
 {
 	unsigned int msecs_in_cmd, msecs_in_ioloop;
 	uint64_t ioloop_wait_usecs;
+	unsigned int msecs_since_cmd;
 
 	if (cmd->start_time.tv_sec == 0)
 		return;
@@ -488,12 +489,19 @@ client_cmd_append_timing_stats(struct client_command_context *cmd,
 	msecs_in_cmd = (cmd->running_usecs + 999) / 1000;
 	msecs_in_ioloop = (ioloop_wait_usecs -
 			   cmd->start_ioloop_wait_usecs + 999) / 1000;
+	msecs_since_cmd = timeval_diff_msecs(&ioloop_timeval,
+					     &cmd->last_run_timeval);
 
 	if (str_data(str)[str_len(str)-1] == '.')
 		str_truncate(str, str_len(str)-1);
-	str_printfa(str, " (%d.%03d + %d.%03d secs).",
+	str_printfa(str, " (%d.%03d + %d.%03d ",
 		    msecs_in_cmd / 1000, msecs_in_cmd % 1000,
 		    msecs_in_ioloop / 1000, msecs_in_ioloop % 1000);
+	if (msecs_since_cmd > 0) {
+		str_printfa(str, "+ %d.%03d ",
+			    msecs_since_cmd / 1000, msecs_since_cmd % 1000);
+	}
+	str_append(str, "secs).");
 }
 
 void client_send_tagline(struct client_command_context *cmd, const char *data)
@@ -721,6 +729,7 @@ struct client_command_context *client_command_alloc(struct client *client)
 	cmd->client = client;
 	cmd->pool = client->command_pool;
 	cmd->start_time = ioloop_timeval;
+	cmd->last_run_timeval = ioloop_timeval;
 	cmd->start_ioloop_wait_usecs = io_loop_get_wait_usecs(current_ioloop);
 	p_array_init(&cmd->module_contexts, cmd->pool, 5);
 
diff --git a/src/imap/imap-client.h b/src/imap/imap-client.h
index 394b59cc19d..9b5415e26f2 100644
--- a/src/imap/imap-client.h
+++ b/src/imap/imap-client.h
@@ -79,6 +79,9 @@ struct client_command_context {
 	/* time when command handling was started - typically this is after
 	   reading all the parameters. */
 	struct timeval start_time;
+	/* time when command handling was last finished. this is before
+	   mailbox syncing is done. */
+	struct timeval last_run_timeval;
 	/* io_loop_get_wait_usecs()'s value when the command was started */
 	uint64_t start_ioloop_wait_usecs;
 	/* how many usecs this command itself has spent running */
diff --git a/src/imap/imap-sync.c b/src/imap/imap-sync.c
index 50ef1f3048a..27f31d05430 100644
--- a/src/imap/imap-sync.c
+++ b/src/imap/imap-sync.c
@@ -762,6 +762,7 @@ bool cmd_sync(struct client_command_context *cmd, enum mailbox_sync_flags flags,
 	if (cmd->cancel)
 		return TRUE;
 
+	cmd->last_run_timeval = ioloop_timeval;
 	if (client->mailbox == NULL) {
 		/* no mailbox selected, no point in delaying the sync */
 		if (tagline != NULL)

From 744fa5af70c19cce03974217302328bbe137d8dd Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Wed, 18 May 2016 09:29:04 +0300
Subject: [PATCH 155/450] dict: Fix crash when no dictionaries defined

---
 src/dict/dict-connection.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/dict/dict-connection.c b/src/dict/dict-connection.c
index 762562a33e4..ae88ae99e7a 100644
--- a/src/dict/dict-connection.c
+++ b/src/dict/dict-connection.c
@@ -76,6 +76,10 @@ static int dict_connection_dict_init(struct dict_connection *conn)
 	unsigned int i, count;
 	const char *uri, *error;
 
+	if (!array_is_created(&dict_settings->dicts)) {
+		i_error("dict client: No dictionaries configured");
+		return -1;
+	}
 	strlist = array_get(&dict_settings->dicts, &count);
 	for (i = 0; i < count; i += 2) {
 		if (strcmp(strlist[i], conn->name) == 0)

From e17d7f4e6396e5345386132d788f3a13c8ae88f5 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Wed, 18 May 2016 09:33:15 +0300
Subject: [PATCH 156/450] doveadm-mailbox-delete: Restore recursive parameter

---
 src/doveadm/doveadm-mail-mailbox.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/doveadm/doveadm-mail-mailbox.c b/src/doveadm/doveadm-mail-mailbox.c
index 2050b24b5bc..c1f7c2ee060 100644
--- a/src/doveadm/doveadm-mail-mailbox.c
+++ b/src/doveadm/doveadm-mail-mailbox.c
@@ -685,6 +685,7 @@ struct doveadm_cmd_ver2 doveadm_cmd_mailbox_delete_ver2 = {
 DOVEADM_CMD_PARAMS_START
 DOVEADM_CMD_MAIL_COMMON
 DOVEADM_CMD_PARAM('s', "subscriptions", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('r', "recursive", CMD_PARAM_BOOL, 0)
 DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
 DOVEADM_CMD_PARAMS_END
 };

From 94e8a82e51c24edafaa9a9e154c4df0150f78a99 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 18 May 2016 18:48:27 +0300
Subject: [PATCH 157/450] lib: If epoll_ctl() fails, panic instead of just
 fatal.

This makes it easier to fix such bugs.
---
 src/lib/ioloop-epoll.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/ioloop-epoll.c b/src/lib/ioloop-epoll.c
index 1143abecb61..c2fe1a94867 100644
--- a/src/lib/ioloop-epoll.c
+++ b/src/lib/ioloop-epoll.c
@@ -105,7 +105,7 @@ void io_loop_handle_add(struct io_file *io)
 
 	if (epoll_ctl(ctx->epfd, op, io->fd, &event) < 0) {
 		if (errno == EPERM && op == EPOLL_CTL_ADD) {
-			i_fatal("epoll_ctl(add, %d) failed: %m "
+			i_panic("epoll_ctl(add, %d) failed: %m "
 				"(fd doesn't support epoll%s)", io->fd,
 				io->fd != STDIN_FILENO ? "" :
 				" - instead of '<file', try 'cat file|'");

From 580cfdf2535cc07436fe6ec75680a1b53026ae46 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 18 May 2016 18:51:55 +0300
Subject: [PATCH 158/450] lib-http: Fixed checking if istream or ostream needs
 waiting.

---
 src/lib-http/http-client-request.c  | 2 +-
 src/lib-http/http-server-response.c | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib-http/http-client-request.c b/src/lib-http/http-client-request.c
index 96120583587..e5ce0083720 100644
--- a/src/lib-http/http-client-request.c
+++ b/src/lib-http/http-client-request.c
@@ -835,7 +835,7 @@ int http_client_request_send_more(struct http_client_request *req,
 			/* finished sending payload */
 			http_client_request_finish_payload_out(req);
 		}
-	} else if (i_stream_get_data_size(req->payload_input) > 0) {
+	} else if (i_stream_have_bytes_left(req->payload_input)) {
 		/* output is blocking (server needs to act; enable timeout) */
 		conn->output_locked = TRUE;
 		if (!pipelined)
diff --git a/src/lib-http/http-server-response.c b/src/lib-http/http-server-response.c
index f8379364740..42f328cd836 100644
--- a/src/lib-http/http-server-response.c
+++ b/src/lib-http/http-server-response.c
@@ -528,7 +528,7 @@ int http_server_response_send_more(struct http_server_response *resp,
 		}
 		/* finished sending payload */
 		http_server_response_finish_payload_out(resp);
-	} else if (i_stream_get_data_size(resp->payload_input) > 0) {
+	} else if (i_stream_have_bytes_left(resp->payload_input)) {
 		/* output is blocking */
 		conn->output_locked = TRUE;
 		o_stream_set_flush_pending(output, TRUE);

From 06bd6c4d4b6d88a1f9eb2723c2924ecda1d536bc Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 18 May 2016 14:56:59 +0300
Subject: [PATCH 159/450] lib-mail: istream-header-filter - Don't return -2 too
 early

We returned -2 after we already added the data to the buffer. We should
have returned -2 only on the next read() call if nothing was skipped.
---
 src/lib-mail/istream-header-filter.c      |  3 --
 src/lib-mail/test-istream-header-filter.c | 55 +++++++++++++++++++++++
 2 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/src/lib-mail/istream-header-filter.c b/src/lib-mail/istream-header-filter.c
index 728ee5d2b63..427f4fed140 100644
--- a/src/lib-mail/istream-header-filter.c
+++ b/src/lib-mail/istream-header-filter.c
@@ -149,9 +149,6 @@ static ssize_t hdr_stream_update_pos(struct header_filter_istream *mstream)
 	ret = (ssize_t)(pos - mstream->istream.pos - mstream->istream.skip);
 	i_assert(ret >= 0);
 	mstream->istream.pos = pos;
-
-	if (pos >= mstream->istream.max_buffer_size)
-		return -2;
 	return ret;
 }
 
diff --git a/src/lib-mail/test-istream-header-filter.c b/src/lib-mail/test-istream-header-filter.c
index 0523390d8cd..d2d0765f6e8 100644
--- a/src/lib-mail/test-istream-header-filter.c
+++ b/src/lib-mail/test-istream-header-filter.c
@@ -200,6 +200,60 @@ static void test_istream_filter_large_buffer(void)
 	test_end();
 }
 
+static void test_istream_filter_large_buffer2(void)
+{
+	static const char *wanted_headers[] = { "References" };
+	string_t *input, *output;
+	struct istream *istream, *filter;
+	const struct stat *st;
+	const unsigned char *data;
+	size_t size;
+	unsigned int i;
+	int ret;
+
+	test_begin("i_stream_create_header_filter: large buffer2");
+
+	input = str_new(default_pool, 1024*128);
+	output = str_new(default_pool, 1024*128);
+	str_append(input, "References: ");
+	add_random_text(input, 1024*64);
+	str_append(input, "\r\n\r\n");
+
+	istream = test_istream_create_data(str_data(input), str_len(input));
+	test_istream_set_max_buffer_size(istream, 8192);
+
+	filter = i_stream_create_header_filter(istream,
+		HEADER_FILTER_INCLUDE | HEADER_FILTER_HIDE_BODY,
+		wanted_headers, N_ELEMENTS(wanted_headers),
+		*null_header_filter_callback, (void *)NULL);
+
+	for (i = 0; i < 2; i++) {
+		while ((ret = i_stream_read_more(filter, &data, &size)) > 0) {
+			str_append_n(output, data, size);
+			i_stream_skip(filter, size);
+		}
+		test_assert(ret == -1);
+		test_assert(filter->stream_errno == 0);
+
+		test_assert(strcmp(str_c(input), str_c(output)) == 0);
+		test_assert(i_stream_stat(filter, TRUE, &st) == 0 &&
+			    (uoff_t)st->st_size == filter->v_offset + size);
+
+		/* seek back and retry once with caching and different
+		   buffer size */
+		i_stream_seek(filter, 0);
+		str_truncate(output, 0);
+		test_istream_set_max_buffer_size(istream, 4096);
+	}
+
+	str_free(&input);
+	str_free(&output);
+	i_stream_unref(&filter);
+	i_stream_unref(&istream);
+
+	test_end();
+}
+
 static void
 filter3_callback(struct header_filter_istream *input ATTR_UNUSED,
 		 struct message_header_line *hdr,
@@ -460,6 +514,7 @@ int main(void)
 	static void (*test_functions[])(void) = {
 		test_istream_filter,
 		test_istream_filter_large_buffer,
+		test_istream_filter_large_buffer2,
 		test_istream_callbacks,
 		test_istream_edit,
 		test_istream_add_missing_eoh,

From 81f0bc16544ab84eebd851a56487bce94f1d2d62 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 18 May 2016 22:26:50 +0300
Subject: [PATCH 160/450] lib-fs: istream-metawrap can't have a readable fd
 currently

---
 src/lib-fs/istream-metawrap.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/lib-fs/istream-metawrap.c b/src/lib-fs/istream-metawrap.c
index c869736278c..d757f51a43b 100644
--- a/src/lib-fs/istream-metawrap.c
+++ b/src/lib-fs/istream-metawrap.c
@@ -132,7 +132,9 @@ i_stream_create_metawrap(struct istream *input,
 	mstream->istream.seek = i_stream_metawrap_seek;
 	mstream->istream.stat = input->seekable ? i_stream_metawrap_stat : NULL;
 
-	mstream->istream.istream.readable_fd = input->readable_fd;
+	/* we can't set abs_start_offset early enough so that it would get
+	   passed to our child istreams. */
+	mstream->istream.istream.readable_fd = FALSE;
 	mstream->istream.istream.blocking = input->blocking;
 	mstream->istream.istream.seekable = input->seekable;
 	mstream->in_metadata = TRUE;

From dca07197ab5bc2ba29f30d6c3814c24f7518d1f2 Mon Sep 17 00:00:00 2001
From: Baofeng Wang <baofeng.wang@dovecot.fi>
Date: Mon, 16 May 2016 11:34:05 +0300
Subject: [PATCH 161/450] config: fix possible out-of-bound access in parsing
 function

Add low-bound check when decreasing string pointer from tail.
---
 src/config/config-parser.c | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/config/config-parser.c b/src/config/config-parser.c
index 2a5009af7a8..c6e04256c48 100644
--- a/src/config/config-parser.c
+++ b/src/config/config-parser.c
@@ -595,17 +595,25 @@ config_parse_line(struct config_parser_context *ctx,
 
 	/* remove whitespace from end of line */
 	len = strlen(line);
-	while (IS_WHITE(line[len-1]))
+	while (len >= 1) {
+		if(!IS_WHITE(line[len-1]))
+			break;
 		len--;
+	}
 	line[len] = '\0';
 
-	if (len > 0 && line[len-1] == '\\') {
+	if (len >= 1 && line[len-1] == '\\') {
 		/* continues in next line */
 		len--;
-		while (IS_WHITE(line[len-1]))
+		while (len >= 1) {
+			if(!IS_WHITE(line[len-1]))
+				break;
 			len--;
-		str_append_n(full_line, line, len);
-		str_append_c(full_line, ' ');
+		}
+		if(len >= 1) {
+			str_append_n(full_line, line, len);
+			str_append_c(full_line, ' ');
+		}
 		return CONFIG_LINE_TYPE_CONTINUE;
 	}
 	if (str_len(full_line) > 0) {

From 104a244966641e13bb18b6a8006180702c88d518 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 19 May 2016 23:19:29 +0300
Subject: [PATCH 162/450] lib-mail: Fixed istream-header-filter calling
 callback(hdr=NULL)

It should do it on the following parsings as well if there were any
callbacks that were called, or if the hdr==NULL callback added any
headers. This is important because the hdr==NULL call could do some
cleanups.
---
 src/lib-mail/istream-header-filter.c      |  10 +-
 src/lib-mail/test-istream-header-filter.c | 146 ++++++++++++++++------
 2 files changed, 115 insertions(+), 41 deletions(-)

diff --git a/src/lib-mail/istream-header-filter.c b/src/lib-mail/istream-header-filter.c
index 427f4fed140..19b671a247c 100644
--- a/src/lib-mail/istream-header-filter.c
+++ b/src/lib-mail/istream-header-filter.c
@@ -41,6 +41,7 @@ struct header_filter_istream {
 	unsigned int last_orig_crlf:1;
 	unsigned int last_added_newline:1;
 	unsigned int eoh_not_matched:1;
+	unsigned int callbacks_called:1;
 	unsigned int prev_matched:1;
 };
 
@@ -203,6 +204,7 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 			} else if (mstream->callback != NULL) {
 				mstream->callback(mstream, hdr, &matched,
 						  mstream->context);
+				mstream->callbacks_called = TRUE;
 			}
 
 			if (!matched) {
@@ -241,6 +243,7 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 			mstream->parsed_lines = mstream->cur_line;
 			mstream->callback(mstream, hdr, &matched,
 					  mstream->context);
+			mstream->callbacks_called = TRUE;
 			if (matched != orig_matched &&
 			    !hdr->continued && !mstream->headers_edited) {
 				if (!array_is_created(&mstream->match_change_lines))
@@ -323,12 +326,14 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 		message_parse_header_deinit(&mstream->hdr_ctx);
 		mstream->hdr_ctx = NULL;
 
-		if (!mstream->header_parsed && mstream->callback != NULL) {
+		if ((!mstream->header_parsed || mstream->headers_edited ||
+		     mstream->callbacks_called) &&
+		    mstream->callback != NULL) {
 			bool matched = FALSE;
 			mstream->callback(mstream, NULL,
 					  &matched, mstream->context);
 			/* check if the callback added more headers.
-			   this is allowed only of EOH wasn't added yet. */
+			   this is allowed only if EOH wasn't added yet. */
 			ret2 = hdr_stream_update_pos(mstream);
 			if (!mstream->seen_eoh)
 				ret += ret2;
@@ -338,6 +343,7 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 		}
 		mstream->header_parsed = TRUE;
 		mstream->header_read = TRUE;
+		mstream->callbacks_called = FALSE;
 
 		mstream->header_size.physical_size =
 			mstream->istream.parent->v_offset;
diff --git a/src/lib-mail/test-istream-header-filter.c b/src/lib-mail/test-istream-header-filter.c
index d2d0765f6e8..46c90ca93fb 100644
--- a/src/lib-mail/test-istream-header-filter.c
+++ b/src/lib-mail/test-istream-header-filter.c
@@ -7,15 +7,39 @@
 #include "istream-header-filter.h"
 #include "test-common.h"
 
+struct run_ctx {
+	header_filter_callback *callback;
+	bool null_hdr_seen;
+	bool callback_called;
+};
+
+static void run_callback(struct header_filter_istream *input,
+			 struct message_header_line *hdr,
+			 bool *matched, struct run_ctx *ctx)
+{
+	if (hdr == NULL)
+		ctx->null_hdr_seen = TRUE;
+	if (ctx->callback != NULL)
+		ctx->callback(input, hdr, matched, NULL);
+	ctx->callback_called = TRUE;
+}
+
 static void
-test_istream_run(struct istream *test_istream, struct istream *filter,
-		 unsigned int input_len, const char *output)
+test_istream_run(struct istream *test_istream,
+		 unsigned int input_len, const char *output,
+		 enum header_filter_flags flags,
+		 header_filter_callback *callback)
 {
+	struct run_ctx run_ctx = { .callback = callback, .null_hdr_seen = FALSE };
+	struct istream *filter;
 	unsigned int i, output_len = strlen(output);
 	const struct stat *st;
 	const unsigned char *data;
 	size_t size;
 
+	filter = i_stream_create_header_filter(test_istream, flags, NULL, 0,
+					       run_callback, &run_ctx);
+
 	for (i = 1; i < input_len; i++) {
 		test_istream_set_size(test_istream, i);
 		test_assert(i_stream_read(filter) >= 0);
@@ -24,6 +48,10 @@ test_istream_run(struct istream *test_istream, struct istream *filter,
 	test_assert(i_stream_read(filter) > 0);
 	test_assert(i_stream_read(filter) == -1);
 
+	test_assert(run_ctx.null_hdr_seen);
+	run_ctx.null_hdr_seen = FALSE;
+	run_ctx.callback_called = FALSE;
+
 	data = i_stream_get_data(filter, &size);
 	test_assert(size == output_len && memcmp(data, output, size) == 0);
 
@@ -31,10 +59,12 @@ test_istream_run(struct istream *test_istream, struct istream *filter,
 	i_stream_skip(filter, size);
 	i_stream_seek(filter, 0);
 	while (i_stream_read(filter) > 0) ;
+	test_assert(run_ctx.null_hdr_seen == run_ctx.callback_called);
 	data = i_stream_get_data(filter, &size);
 	test_assert(size == output_len && memcmp(data, output, size) == 0);
 	test_assert(i_stream_stat(filter, TRUE, &st) == 0 &&
 		    (uoff_t)st->st_size == size);
+	i_stream_unref(&filter);
 }
 
 static void ATTR_NULL(3)
@@ -118,9 +148,11 @@ static void add_random_text(string_t *dest, unsigned int count)
 static void ATTR_NULL(3)
 filter2_callback(struct header_filter_istream *input ATTR_UNUSED,
 		 struct message_header_line *hdr,
-		 bool *matched, void *context ATTR_UNUSED)
+		 bool *matched, bool *null_hdr_seen)
 {
-	if (hdr != NULL && strcmp(hdr->name, "To") == 0)
+	if (hdr == NULL)
+		*null_hdr_seen = TRUE;
+	else if (strcmp(hdr->name, "To") == 0)
 		*matched = TRUE;
 }
 
@@ -133,6 +165,7 @@ static void test_istream_filter_large_buffer(void)
 	size_t size, prefix_len;
 	const char *p;
 	unsigned int i;
+	bool null_hdr_seen = FALSE;
 
 	test_begin("i_stream_create_header_filter: large buffer");
 
@@ -154,7 +187,7 @@ static void test_istream_filter_large_buffer(void)
 					       HEADER_FILTER_NO_CR,
 					       NULL, 0,
 					       filter2_callback,
-					       (void *)NULL);
+					       &null_hdr_seen);
 
 	for (i = 0; i < 2; i++) {
 		for (;;) {
@@ -168,6 +201,8 @@ static void test_istream_filter_large_buffer(void)
 				i_stream_skip(filter, size);
 			}
 		}
+		/* callbacks are called only once */
+		test_assert(null_hdr_seen == (i == 0));
 
 		data = i_stream_get_data(filter, &size);
 		test_assert(size <= 8192);
@@ -190,6 +225,7 @@ static void test_istream_filter_large_buffer(void)
 		i_stream_seek(filter, 0);
 		str_truncate(output, 0);
 		test_istream_set_max_buffer_size(istream, 4096);
+		null_hdr_seen = FALSE;
 	}
 
 	str_free(&input);
@@ -333,17 +369,14 @@ static void test_istream_edit(void)
 {
 	const char *input = "From: foo\nTo: bar\n\nhello world\n";
 	const char *output = "From: foo\nTo: 123\nAdded: header\n\nhello world\n";
-	struct istream *istream, *filter;
+	struct istream *istream;
 
 	test_begin("i_stream_create_header_filter: edit headers");
 	istream = test_istream_create(input);
-	filter = i_stream_create_header_filter(istream,
-					       HEADER_FILTER_EXCLUDE |
-					       HEADER_FILTER_NO_CR,
-					       NULL, 0,
-					       edit_callback, (void *)NULL);
-	test_istream_run(istream, filter, strlen(input), output);
-	i_stream_unref(&filter);
+	test_istream_run(istream, strlen(input), output,
+			 HEADER_FILTER_EXCLUDE |
+			 HEADER_FILTER_NO_CR,
+			 edit_callback);
 	i_stream_unref(&istream);
 
 	test_end();
@@ -425,22 +458,19 @@ static void test_istream_add_missing_eoh(void)
 		{ "From: foo\r\n\r\n", "From: foo\r\n\r\n", 0 },
 		{ "From: foo\r\n\r\nbar", "From: foo\r\n\r\nbar", 0 }
 	};
-	struct istream *istream, *filter;
+	struct istream *istream;
 	unsigned int i;
 
 	test_begin("i_stream_create_header_filter: add missing EOH");
 	for (i = 0; i < N_ELEMENTS(tests); i++) {
 		istream = test_istream_create(tests[i].input);
-		filter = i_stream_create_header_filter(istream,
-					       HEADER_FILTER_EXCLUDE |
-					       HEADER_FILTER_CRLF_PRESERVE |
-					       HEADER_FILTER_ADD_MISSING_EOH,
-					       NULL, 0,
-					       *null_header_filter_callback, (void *)NULL);
-		test_istream_run(istream, filter,
+		test_istream_run(istream,
 				 strlen(tests[i].input) + tests[i].extra,
-				 tests[i].output);
-		i_stream_unref(&filter);
+				 tests[i].output,
+				 HEADER_FILTER_EXCLUDE |
+				 HEADER_FILTER_CRLF_PRESERVE |
+				 HEADER_FILTER_ADD_MISSING_EOH,
+				 *null_header_filter_callback);
 		i_stream_unref(&istream);
 	}
 	test_end();
@@ -461,22 +491,19 @@ static void test_istream_hide_body(void)
 		{ "From: foo\r\n\r\n", "From: foo\r\n\r\n", 0 },
 		{ "From: foo\r\n\r\nbar", "From: foo\r\n\r\n", -3 }
 	};
-	struct istream *istream, *filter;
+	struct istream *istream;
 	unsigned int i;
 
 	test_begin("i_stream_create_header_filter: hide body");
 	for (i = 0; i < N_ELEMENTS(tests); i++) {
 		istream = test_istream_create(tests[i].input);
-		filter = i_stream_create_header_filter(istream,
-					       HEADER_FILTER_EXCLUDE |
-					       HEADER_FILTER_CRLF_PRESERVE |
-					       HEADER_FILTER_HIDE_BODY,
-					       NULL, 0,
-					       *null_header_filter_callback, (void *)NULL);
-		test_istream_run(istream, filter,
+		test_istream_run(istream,
 				 strlen(tests[i].input) + tests[i].extra,
-				 tests[i].output);
-		i_stream_unref(&filter);
+				 tests[i].output,
+				 HEADER_FILTER_EXCLUDE |
+				 HEADER_FILTER_CRLF_PRESERVE |
+				 HEADER_FILTER_HIDE_BODY,
+				 *null_header_filter_callback);
 		i_stream_unref(&istream);
 	}
 	test_end();
@@ -495,17 +522,56 @@ static void test_istream_strip_eoh(void)
 {
 	const char *input = "From: foo\nTo: bar\n\nhello world\n";
 	const char *output = "From: foo\nTo: bar\nhello world\n";
-	struct istream *istream, *filter;
+	struct istream *istream;
 
 	test_begin("i_stream_create_header_filter: strip_eoh");
 	istream = test_istream_create(input);
-	filter = i_stream_create_header_filter(istream,
-			HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR, NULL, 0,
-			strip_eoh_callback, (void *)NULL);
-	test_istream_run(istream, filter, strlen(input), output);
-	i_stream_unref(&filter);
+	test_istream_run(istream, strlen(input), output,
+			 HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR,
+			 strip_eoh_callback);
+	i_stream_unref(&istream);
+
+	test_end();
+}
+
+static void ATTR_NULL(3)
+missing_eoh_callback(struct header_filter_istream *input ATTR_UNUSED,
+		     struct message_header_line *hdr,
+		     bool *matched ATTR_UNUSED, void *context ATTR_UNUSED)
+{
+	if (hdr == NULL) {
+		const char *new_hdr = "Subject: added\n\n";
+		i_stream_header_filter_add(input, new_hdr, strlen(new_hdr));
+	}
+}
+
+static void test_istream_missing_eoh_callback(void)
+{
+	const char *input = "From: foo\nTo: bar\n";
+	const char *output = "From: foo\nTo: bar\nSubject: added\n\n";
+	struct istream *istream;
+
+	test_begin("i_stream_create_header_filter: add headers when EOH is missing");
+	istream = test_istream_create(input);
+	test_istream_run(istream, strlen(input) + 1, output,
+			 HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR,
+			 missing_eoh_callback);
 	i_stream_unref(&istream);
+	test_end();
+}
+
+static void test_istream_empty_missing_eoh_callback(void)
+{
+	const char *input = "";
+	const char *output = "Subject: added\n\n";
+	struct istream *istream;
 
+	test_begin("i_stream_create_header_filter: add headers when mail is empty");
+	istream = test_istream_create(input);
+	test_istream_run(istream, strlen(input)+1, output,
+			 HEADER_FILTER_EXCLUDE | HEADER_FILTER_NO_CR,
+			 missing_eoh_callback);
+	i_stream_unref(&istream);
 	test_end();
 }
 
@@ -521,6 +587,8 @@ int main(void)
 		test_istream_end_body_with_lf,
 		test_istream_hide_body,
 		test_istream_strip_eoh,
+		test_istream_missing_eoh_callback,
+		test_istream_empty_missing_eoh_callback,
 		NULL
 	};
 	return test_run(test_functions);

From 153528ae66c742689c41d3f4279a15feb9f130d0 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 20 May 2016 15:55:52 +0300
Subject: [PATCH 163/450] doveadm backup: Fixed unnecessary mailbox deletion.

Broken by 79490ec1a. The idea is to delete the mailbox early to avoid
unnecessary work, but we deleted it too early. If the last mail(s) were
deleted instead of just expunging them we deleted the whole mailbox.
---
 src/doveadm/dsync/dsync-mailbox-import.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c
index 3392a653492..6a44f88a464 100644
--- a/src/doveadm/dsync/dsync-mailbox-import.c
+++ b/src/doveadm/dsync/dsync-mailbox-import.c
@@ -785,9 +785,10 @@ static bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer,
 		/* add a record for local mail */
 		i_assert(importer->cur_mail != NULL);
 		if (importer->revert_local_changes) {
-			if (save_change == NULL) {
+			if (save_change == NULL &&
+			    importer->cur_mail->uid >= importer->remote_uid_next) {
 				dsync_mailbox_revert_existing_uid(importer, importer->cur_mail->uid,
-					t_strdup_printf("highest than remote's UIDs (remote UIDNEXT=%u)", importer->remote_uid_next));
+					t_strdup_printf("higher than remote's UIDs (remote UIDNEXT=%u)", importer->remote_uid_next));
 				return TRUE;
 			}
 			mail_expunge(importer->cur_mail);

From 20e68d86f08b614c77ade09163fbb4d4654d25ef Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 23 May 2016 13:08:20 +0300
Subject: [PATCH 164/450] lib-ldap: Support libldap without
 LDAP_OPT_X_TLS_NEWCTX.

---
 src/lib-ldap/ldap-connection.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/lib-ldap/ldap-connection.c b/src/lib-ldap/ldap-connection.c
index 1322c9341d4..be9f89b33ed 100644
--- a/src/lib-ldap/ldap-connection.c
+++ b/src/lib-ldap/ldap-connection.c
@@ -88,8 +88,10 @@ int ldap_connection_setup(struct ldap_connection *conn, const char **error_r)
 
 	ldap_set_option(conn->conn, LDAP_OPT_REFERRALS, 0);
 
+#ifdef LDAP_OPT_X_TLS_NEWCTX
 	opt = 0;
 	ldap_set_option(conn->conn, LDAP_OPT_X_TLS_NEWCTX, &opt);
+#endif
 
 	return 0;
 }

From 24234b1714ee1ae04b9b55b11e7916d75766929f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 23 May 2016 13:12:41 +0300
Subject: [PATCH 165/450] lib-ldap: Use ldap_unbind_ext() instead of
 ldap_destroy()

This allows it to compile with older LDAP libraries. Their behavior (at least
in OpenLDAP) is identical as long as LDAP_OPT_CLIENT_CONTROLS haven't been
changed, which we haven't.
---
 src/lib-ldap/ldap-connection.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-ldap/ldap-connection.c b/src/lib-ldap/ldap-connection.c
index be9f89b33ed..31a27cf5139 100644
--- a/src/lib-ldap/ldap-connection.c
+++ b/src/lib-ldap/ldap-connection.c
@@ -572,7 +572,7 @@ void ldap_connection_kill(struct ldap_connection *conn)
 		}
 	}
 	if (conn->conn != NULL) {
-		ldap_destroy(conn->conn);
+		ldap_unbind_ext(conn->conn, NULL, NULL);
 		ldap_memfree(conn->scred);
 	}
 	conn->conn = NULL;

From 26c937b6c6671c99a2b4ce00004189ae4bdd5e87 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 23 May 2016 18:46:09 +0300
Subject: [PATCH 166/450] lib-index: Make sure cache compression doesn't clear
 the file unneededly.

This probably never happened with current code, except when the cache file
was already broken.

mail_cache_compress() unmapped cache when map_with_read==TRUE, but we never
mapped it back. This would cause cache copying code to think cache was
unusable and not copy anything.
---
 src/lib-index/mail-cache-compress.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/lib-index/mail-cache-compress.c b/src/lib-index/mail-cache-compress.c
index cfc33de57c8..66bbab4972a 100644
--- a/src/lib-index/mail-cache-compress.c
+++ b/src/lib-index/mail-cache-compress.c
@@ -452,6 +452,9 @@ static int mail_cache_compress_locked(struct mail_cache *cache,
 
 		return mail_cache_reopen(cache) < 0 ? -1 : 0;
 	}
+	/* make sure we have mapped it before reading. */
+	if (mail_cache_map(cache, 0, 0, &data) < 0)
+		return -1;
 
 	/* we want to recreate the cache. write it first to a temporary file */
 	fd = mail_index_create_tmp_file(cache->index, cache->filepath, &temp_path);

From 361575245e2267b7da3ebf0af616945dddf4aeb2 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 23 May 2016 23:24:31 +0300
Subject: [PATCH 167/450] lib-index: Fixed assert-crash caused by previous
 cache compression change.

We can't call mail_cache_map() when cache file isn't open or we'll crash:

Panic: file mail-cache.c: line 468 (mail_cache_map): assertion failed: (cache->need_compress_file_seq != 0 || MAIL_INDEX_IS_IN_MEMORY(cache->index))
---
 src/lib-index/mail-cache-compress.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/lib-index/mail-cache-compress.c b/src/lib-index/mail-cache-compress.c
index 66bbab4972a..1d47b22d97d 100644
--- a/src/lib-index/mail-cache-compress.c
+++ b/src/lib-index/mail-cache-compress.c
@@ -452,9 +452,11 @@ static int mail_cache_compress_locked(struct mail_cache *cache,
 
 		return mail_cache_reopen(cache) < 0 ? -1 : 0;
 	}
-	/* make sure we have mapped it before reading. */
-	if (mail_cache_map(cache, 0, 0, &data) < 0)
-		return -1;
+	if (cache->fd != -1) {
+		/* make sure we have mapped it before reading. */
+		if (mail_cache_map(cache, 0, 0, &data) < 0)
+			return -1;
+	}
 
 	/* we want to recreate the cache. write it first to a temporary file */
 	fd = mail_index_create_tmp_file(cache->index, cache->filepath, &temp_path);

From bf5c1acf579d10d9281af96e2f9efd21e7d45e9e Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 24 May 2016 15:52:37 +0300
Subject: [PATCH 168/450] lib-stats: Allow unregistering + re-registering
 stats.

Fixes:
Panic: stats_register() called after stats_alloc_size() was already called - this will break existing allocations

This could have happened with doveadm HTTP API when using multiple commands
in same HTTP connection.
---
 src/lib-stats/stats.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/lib-stats/stats.c b/src/lib-stats/stats.c
index 258c097f357..30588035e86 100644
--- a/src/lib-stats/stats.c
+++ b/src/lib-stats/stats.c
@@ -68,8 +68,12 @@ void stats_unregister(struct stats_item **_item)
 	array_delete(&stats_items, idx, 1);
 
 	i_free(item);
-	if (array_count(&stats_items) == 0)
+	if (array_count(&stats_items) == 0) {
 		array_free(&stats_items);
+		/* all stats should have been freed by now. allow
+		   re-registering and using stats. */
+		stats_allocated = FALSE;
+	}
 }
 
 struct stats *stats_alloc(pool_t pool)

From bde40af690a254a5e2e00d655bd85bbb5df40fb9 Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@cpanel.net>
Date: Thu, 19 May 2016 19:15:49 -0500
Subject: [PATCH 169/450] dsync: race condition where done/finish is received
 after one side has closed

We do not tell the remote we are closing if they have
already told us because they close the
connection after sending ITEM_FINISH or ITEM_DONE and will
not be ever receive anything else from us unless
it just happens to get combined into the same packet
as a previous response and is already in the buffer.
---
 src/doveadm/dsync/dsync-ibc-stream.c | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/doveadm/dsync/dsync-ibc-stream.c b/src/doveadm/dsync/dsync-ibc-stream.c
index 526f9d5bf50..102fffe855c 100644
--- a/src/doveadm/dsync/dsync-ibc-stream.c
+++ b/src/doveadm/dsync/dsync-ibc-stream.c
@@ -166,6 +166,8 @@ struct dsync_ibc_stream {
 	unsigned int version_received:1;
 	unsigned int handshake_received:1;
 	unsigned int has_pending_data:1;
+	unsigned int finish_received:1;
+	unsigned int done_received:1;
 	unsigned int stopped:1;
 };
 
@@ -366,10 +368,22 @@ static void dsync_ibc_stream_deinit(struct dsync_ibc *_ibc)
 	if (ibc->value_output != NULL)
 		i_stream_unref(&ibc->value_output);
 	else {
-		/* notify remote that we're closing. this is mainly to avoid
-		   "read() failed: EOF" errors on failing dsyncs */
-		o_stream_nsend_str(ibc->output,
-			t_strdup_printf("%c\n", items[ITEM_DONE].chr));
+		/* If the remote has not told us that they are closing we
+		   notify remote that we're closing. this is mainly to avoid
+		   "read() failed: EOF" errors on failing dsyncs.
+
+		   Avoid a race condition:
+		   We do not tell the remote we are closing if they have
+		   already told us because they close the
+		   connection after sending ITEM_DONE and will
+		   not be ever receive anything else from us unless
+		   it just happens to get combined into the same packet
+		   as a previous response and is already in the buffer.
+		*/
+		if (!ibc->done_received && !ibc->finish_received) {
+			o_stream_nsend_str(ibc->output,
+				t_strdup_printf("%c\n", items[ITEM_DONE].chr));
+		}
 		(void)o_stream_nfinish(ibc->output);
 	}
 
@@ -594,6 +608,7 @@ dsync_ibc_stream_input_next(struct dsync_ibc_stream *ibc, enum item_type item,
 		/* remote cleanly closed the connection, possibly because of
 		   some failure (which it should have logged). we don't want to
 		   log any stream errors anyway after this. */
+		ibc->done_received = TRUE;
 		dsync_ibc_stream_stop(ibc);
 		return DSYNC_IBC_RECV_RET_TRYAGAIN;
 
@@ -1939,6 +1954,8 @@ dsync_ibc_stream_recv_finish(struct dsync_ibc *_ibc, const char **error_r,
 		return DSYNC_IBC_RECV_RET_TRYAGAIN;
 	}
 	*mail_error_r = i;
+
+	ibc->finish_received = TRUE;
 	return DSYNC_IBC_RECV_RET_OK;
 }
 

From 801990c1c26ad030e817d97f5ef49f52c14cd1ba Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 25 May 2016 19:45:58 +0300
Subject: [PATCH 170/450] lib: Added timing_reset()

---
 src/lib/test-timing.c | 3 +++
 src/lib/timing.c      | 5 +++++
 src/lib/timing.h      | 3 +++
 3 files changed, 11 insertions(+)

diff --git a/src/lib/test-timing.c b/src/lib/test-timing.c
index e83df9f9bcb..467e47d3948 100644
--- a/src/lib/test-timing.c
+++ b/src/lib/test-timing.c
@@ -71,6 +71,9 @@ void test_timing(void)
 			timing_add_usecs(t, test_inputs[i][j]);
 			test_timing_verify(t, test_inputs[i], j+1);
 		}
+		timing_reset(t);
+		test_assert(timing_get_count(t) == 0);
+		test_assert(timing_get_max(t) == 0);
 		timing_deinit(&t);
 		test_end();
 	}
diff --git a/src/lib/timing.c b/src/lib/timing.c
index 18838c474e2..9b8e2354fa2 100644
--- a/src/lib/timing.c
+++ b/src/lib/timing.c
@@ -27,6 +27,11 @@ void timing_deinit(struct timing **_timing)
 	i_free_and_null(*_timing);
 }
 
+void timing_reset(struct timing *timing)
+{
+	memset(timing, 0, sizeof(*timing));
+}
+
 void timing_add_usecs(struct timing *timing, uint64_t usecs)
 {
 	if (timing->count < TIMING_SUBSAMPLING_BUFFER) {
diff --git a/src/lib/timing.h b/src/lib/timing.h
index 413dc2ed001..79536f9ed19 100644
--- a/src/lib/timing.h
+++ b/src/lib/timing.h
@@ -4,6 +4,9 @@
 struct timing *timing_init(void);
 void timing_deinit(struct timing **timing);
 
+/* Reset all events. */
+void timing_reset(struct timing *timing);
+
 /* Add a new event that took the specified number of usecs. */
 void timing_add_usecs(struct timing *timing, uint64_t usecs);
 

From c9adb47c376afd9115970659072ee07c416e590d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 25 May 2016 19:56:47 +0300
Subject: [PATCH 171/450] lib: Fixed crashes in timing_get_median() and
 timing_get_95th()

These were writing outside allocated memory. They weren't actually used
anywhere yet though.
---
 src/lib/test-timing.c | 15 +++++++++++++++
 src/lib/timing.c      |  6 +++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/src/lib/test-timing.c b/src/lib/test-timing.c
index 467e47d3948..c79d144dc39 100644
--- a/src/lib/test-timing.c
+++ b/src/lib/test-timing.c
@@ -77,4 +77,19 @@ void test_timing(void)
 		timing_deinit(&t);
 		test_end();
 	}
+
+	test_begin("timings large");
+	t = timing_init();
+	for (i = 0; i < 10000; i++)
+		timing_add_usecs(t, i);
+	test_assert(timing_get_count(t) == i);
+	test_assert(timing_get_sum(t) == (i-1)*i/2);
+	test_assert(timing_get_min(t) == 0);
+	test_assert(timing_get_max(t) == i-1);
+	test_assert(timing_get_avg(t) == i/2);
+	/* just test that these work: */
+	test_assert(timing_get_median(t) > 0 && timing_get_median(t) < i-1);
+	test_assert(timing_get_95th(t) > 0 && timing_get_95th(t) < i-1);
+	timing_deinit(&t);
+	test_end();
 }
diff --git a/src/lib/timing.c b/src/lib/timing.c
index 9b8e2354fa2..604f102a3f4 100644
--- a/src/lib/timing.c
+++ b/src/lib/timing.c
@@ -90,7 +90,11 @@ static void timing_ensure_sorted(struct timing *timing)
 {
 	if (timing->sorted)
 		return;
-	i_qsort(timing->samples, timing->count, sizeof(*timing->samples),
+
+	unsigned int count = (timing->count < TIMING_SUBSAMPLING_BUFFER)
+		? timing->count
+		: TIMING_SUBSAMPLING_BUFFER;
+	i_qsort(timing->samples, count, sizeof(*timing->samples),
 		uint64_cmp);
 	timing->sorted = TRUE;
 }

From cfc6d8e5c2424d8013ab000a10ba7bab00794369 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 25 May 2016 20:00:01 +0300
Subject: [PATCH 172/450] dict: Show command statistics in process title.

---
 src/dict/Makefile.am       |  3 +-
 src/dict/dict-commands.c   | 43 ++++++++++++++++++++++++++++-
 src/dict/dict-commands.h   | 11 ++++++++
 src/dict/dict-connection.c | 11 ++++++++
 src/dict/dict-connection.h |  1 +
 src/dict/dict-settings.c   |  4 +++
 src/dict/dict-settings.h   |  2 ++
 src/dict/main.c            | 56 ++++++++++++++++++++++++++++++++++++--
 8 files changed, 127 insertions(+), 4 deletions(-)

diff --git a/src/dict/Makefile.am b/src/dict/Makefile.am
index 93e7a42363a..4606de89020 100644
--- a/src/dict/Makefile.am
+++ b/src/dict/Makefile.am
@@ -33,4 +33,5 @@ dict_SOURCES = \
 noinst_HEADERS = \
 	dict-connection.h \
 	dict-commands.h \
-	dict-settings.h
+	dict-settings.h \
+	main.h
diff --git a/src/dict/dict-commands.c b/src/dict/dict-commands.c
index 9742a5ead74..45078144ca5 100644
--- a/src/dict/dict-commands.c
+++ b/src/dict/dict-commands.c
@@ -5,11 +5,13 @@
 #include "ostream.h"
 #include "str.h"
 #include "strescape.h"
+#include "timing.h"
+#include "time-util.h"
 #include "dict-client.h"
 #include "dict-settings.h"
 #include "dict-connection.h"
 #include "dict-commands.h"
-
+#include "main.h"
 
 #define DICT_OUTPUT_OPTIMAL_SIZE 1024
 
@@ -21,6 +23,7 @@ struct dict_cmd_func {
 struct dict_connection_cmd {
 	const struct dict_cmd_func *cmd;
 	struct dict_connection *conn;
+	struct timeval start_timeval;
 	char *reply;
 
 	struct dict_iterate_context *iter;
@@ -29,6 +32,8 @@ struct dict_connection_cmd {
 	unsigned int trans_id;
 };
 
+struct dict_command_stats cmd_stats;
+
 static void dict_connection_cmd_output_more(struct dict_connection_cmd *cmd);
 
 static void dict_connection_cmd_free(struct dict_connection_cmd *cmd)
@@ -78,11 +83,29 @@ static void dict_connection_cmds_flush(struct dict_connection *conn)
 	dict_connection_unref_safe(conn);
 }
 
+static void
+cmd_stats_update(struct dict_connection_cmd *cmd, struct timing *timing)
+{
+	long long diff;
+
+	if (!dict_settings->verbose_proctitle)
+		return;
+
+	io_loop_time_refresh();
+	diff = timeval_diff_usecs(&ioloop_timeval, &cmd->start_timeval);
+	if (diff < 0)
+		diff = 0;
+	timing_add_usecs(timing, diff);
+	dict_proctitle_update_later();
+}
+
 static void
 cmd_lookup_callback(const struct dict_lookup_result *result, void *context)
 {
 	struct dict_connection_cmd *cmd = context;
 
+	cmd_stats_update(cmd, cmd_stats.lookups);
+
 	if (result->ret > 0) {
 		cmd->reply = i_strdup_printf("%c%s\n",
 			DICT_PROTOCOL_REPLY_OK, str_tabescape(result->value));
@@ -142,6 +165,7 @@ static int cmd_iterate_flush(struct dict_connection_cmd *cmd)
 	str_append_c(str, '\n');
 	o_stream_uncork(cmd->conn->output);
 
+	cmd_stats_update(cmd, cmd_stats.iterations);
 	cmd->reply = i_strdup(str_c(str));
 	dict_connection_cmds_flush(cmd->conn);
 	return 1;
@@ -258,6 +282,8 @@ cmd_commit_finish(struct dict_connection_cmd *cmd, int ret, bool async)
 {
 	char chr;
 
+	cmd_stats_update(cmd, cmd_stats.commits);
+
 	switch (ret) {
 	case 1:
 		chr = DICT_PROTOCOL_REPLY_OK;
@@ -448,6 +474,7 @@ int dict_command_input(struct dict_connection *conn, const char *line)
 	cmd = i_new(struct dict_connection_cmd, 1);
 	cmd->conn = conn;
 	cmd->cmd = cmd_func;
+	cmd->start_timeval = ioloop_timeval;
 	array_append(&conn->cmds, &cmd, 1);
 	dict_connection_ref(conn);
 	if ((ret = cmd_func->func(cmd, line + 1)) <= 0) {
@@ -485,3 +512,17 @@ void dict_connection_cmds_output_more(struct dict_connection *conn)
 		/* cmd should be freed now */
 	}
 }
+
+void dict_commands_init(void)
+{
+	cmd_stats.lookups = timing_init();
+	cmd_stats.iterations = timing_init();
+	cmd_stats.commits = timing_init();
+}
+
+void dict_commands_deinit(void)
+{
+	timing_deinit(&cmd_stats.lookups);
+	timing_deinit(&cmd_stats.iterations);
+	timing_deinit(&cmd_stats.commits);
+}
diff --git a/src/dict/dict-commands.h b/src/dict/dict-commands.h
index 03214c4e520..62200d0bbc4 100644
--- a/src/dict/dict-commands.h
+++ b/src/dict/dict-commands.h
@@ -3,8 +3,19 @@
 
 struct dict_connection;
 
+struct dict_command_stats {
+	struct timing *lookups;
+	struct timing *iterations;
+	struct timing *commits;
+};
+
+extern struct dict_command_stats cmd_stats;
+
 int dict_command_input(struct dict_connection *conn, const char *line);
 
 void dict_connection_cmds_output_more(struct dict_connection *conn);
 
+void dict_commands_init(void);
+void dict_commands_deinit(void);
+
 #endif
diff --git a/src/dict/dict-connection.c b/src/dict/dict-connection.c
index ae88ae99e7a..35d8e9386b9 100644
--- a/src/dict/dict-connection.c
+++ b/src/dict/dict-connection.c
@@ -17,6 +17,7 @@
 #define DICT_CONN_MAX_PENDING_COMMANDS 5
 
 static struct dict_connection *dict_connections;
+static unsigned int dict_connections_count = 0;
 
 static int dict_connection_parse_handshake(struct dict_connection *conn,
 					   const char *line)
@@ -195,6 +196,8 @@ struct dict_connection *dict_connection_create(int fd)
 	o_stream_set_flush_callback(conn->output, dict_connection_output, conn);
 	conn->io = io_add(fd, IO_READ, dict_connection_input, conn);
 	i_array_init(&conn->cmds, DICT_CONN_MAX_PENDING_COMMANDS);
+
+	dict_connections_count++;
 	DLLIST_PREPEND(&dict_connections, conn);
 	return conn;
 }
@@ -269,6 +272,9 @@ void dict_connection_destroy(struct dict_connection *conn)
 	i_assert(!conn->destroyed);
 	i_assert(conn->to_unref == NULL);
 
+	i_assert(dict_connections_count > 0);
+	dict_connections_count--;
+
 	conn->destroyed = TRUE;
 	DLLIST_REMOVE(&dict_connections, conn);
 
@@ -293,6 +299,11 @@ void dict_connection_destroy(struct dict_connection *conn)
 	dict_connection_unref(conn);
 }
 
+unsigned int dict_connections_current_count(void)
+{
+	return dict_connections_count;
+}
+
 void dict_connections_destroy_all(void)
 {
 	while (dict_connections != NULL)
diff --git a/src/dict/dict-connection.h b/src/dict/dict-connection.h
index 87187993128..2f9149c0f05 100644
--- a/src/dict/dict-connection.h
+++ b/src/dict/dict-connection.h
@@ -43,6 +43,7 @@ void dict_connection_unref_safe(struct dict_connection *conn);
 
 void dict_connection_continue_input(struct dict_connection *conn);
 
+unsigned int dict_connections_current_count(void);
 void dict_connections_destroy_all(void);
 
 #endif
diff --git a/src/dict/dict-settings.c b/src/dict/dict-settings.c
index 53a27e3c179..6b36cac62bb 100644
--- a/src/dict/dict-settings.c
+++ b/src/dict/dict-settings.c
@@ -86,6 +86,8 @@ struct service_settings dict_async_service_settings = {
 
 static const struct setting_define dict_setting_defines[] = {
 	DEF(SET_STR, base_dir),
+	DEF(SET_BOOL, verbose_proctitle),
+
 	DEF(SET_STR, dict_db_config),
 	{ SET_STRLIST, "dict", offsetof(struct dict_server_settings, dicts), NULL },
 
@@ -94,6 +96,8 @@ static const struct setting_define dict_setting_defines[] = {
 
 const struct dict_server_settings dict_default_settings = {
 	.base_dir = PKG_RUNDIR,
+	.verbose_proctitle = FALSE,
+
 	.dict_db_config = "",
 	.dicts = ARRAY_INIT
 };
diff --git a/src/dict/dict-settings.h b/src/dict/dict-settings.h
index 0068ef04a46..3c37589e9be 100644
--- a/src/dict/dict-settings.h
+++ b/src/dict/dict-settings.h
@@ -3,6 +3,8 @@
 
 struct dict_server_settings {
 	const char *base_dir;
+	bool verbose_proctitle;
+
 	const char *dict_db_config;
 	ARRAY(const char *) dicts;
 };
diff --git a/src/dict/main.c b/src/dict/main.c
index e6c945eb918..f1d89fe27f9 100644
--- a/src/dict/main.c
+++ b/src/dict/main.c
@@ -2,8 +2,12 @@
 
 #include "lib.h"
 #include "restrict-access.h"
+#include "ioloop.h"
 #include "randgen.h"
+#include "str.h"
 #include "hostpid.h"
+#include "timing.h"
+#include "process-title.h"
 #include "env-util.h"
 #include "module-dir.h"
 #include "master-service.h"
@@ -11,10 +15,54 @@
 #include "sql-api.h"
 #include "dict.h"
 #include "dict-client.h"
+#include "dict-commands.h"
 #include "dict-connection.h"
 #include "dict-settings.h"
+#include "main.h"
 
 static struct module *modules;
+static struct timeout *to_proctitle;
+static bool proctitle_updated;
+
+static void
+add_timing_string(string_t *str, struct timing *timing, const char *name)
+{
+	str_printfa(str, ", %u %s:%llu/%llu/%llu/%llu",
+		    timing_get_count(timing), name,
+		    (unsigned long long)timing_get_min(timing)/1000,
+		    (unsigned long long)timing_get_avg(timing)/1000,
+		    (unsigned long long)timing_get_95th(timing)/1000,
+		    (unsigned long long)timing_get_max(timing)/1000);
+	timing_reset(timing);
+}
+
+static void dict_proctitle_update(void *context ATTR_UNUSED)
+{
+	string_t *str = t_str_new(128);
+
+	if (!proctitle_updated)
+		timeout_remove(&to_proctitle);
+
+	str_printfa(str, "[%u clients", dict_connections_current_count());
+
+	add_timing_string(str, cmd_stats.lookups, "lookups");
+	add_timing_string(str, cmd_stats.iterations, "iters");
+	add_timing_string(str, cmd_stats.commits, "commits");
+	str_append_c(str, ']');
+
+	process_title_set(str_c(str));
+	proctitle_updated = FALSE;
+}
+
+void dict_proctitle_update_later(void)
+{
+	if (!dict_settings->verbose_proctitle)
+		return;
+
+	if (to_proctitle == NULL)
+		to_proctitle = timeout_add(1000, dict_proctitle_update, NULL);
+	proctitle_updated = TRUE;
+}
 
 static void dict_die(void)
 {
@@ -68,12 +116,17 @@ static void main_init(void)
 	/* Register only after loading modules. They may contain SQL drivers,
 	   which we'll need to register. */
 	dict_drivers_register_all();
+	dict_commands_init();
 }
 
 static void main_deinit(void)
 {
+	if (to_proctitle != NULL)
+		timeout_remove(&to_proctitle);
+
 	dict_connections_destroy_all();
 	dict_drivers_unregister_all();
+	dict_commands_deinit();
 
 	module_dir_unload(&modules);
 
@@ -83,8 +136,7 @@ static void main_deinit(void)
 
 int main(int argc, char *argv[])
 {
-	const enum master_service_flags service_flags =
-		MASTER_SERVICE_FLAG_UPDATE_PROCTITLE;
+	const enum master_service_flags service_flags = 0;
 	const struct setting_parser_info *set_roots[] = {
 		&dict_setting_parser_info,
 		NULL

From 17aed59dc9d90d78b684d75f96e59f4eb90205ed Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 25 May 2016 22:08:04 +0300
Subject: [PATCH 173/450] dict: Added missing main.h from previous commit

---
 src/dict/main.h | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 src/dict/main.h

diff --git a/src/dict/main.h b/src/dict/main.h
new file mode 100644
index 00000000000..e505e134c89
--- /dev/null
+++ b/src/dict/main.h
@@ -0,0 +1,6 @@
+#ifndef MAIN_H
+#define MAIN_H
+
+void dict_proctitle_update_later(void);
+
+#endif

From a2f421418658fca58cfa0f6a2ae77b730992cb7f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 25 May 2016 22:29:47 +0300
Subject: [PATCH 174/450] cassandra: Added metrics=path connect setting.

Cassandra's metrics are written to the path in JSON format. It can be a file
or a FIFO. The path supports expanding the standard global %variables, such
as %{pid}
---
 src/lib-sql/driver-cassandra.c | 76 ++++++++++++++++++++++++++++++++++
 1 file changed, 76 insertions(+)

diff --git a/src/lib-sql/driver-cassandra.c b/src/lib-sql/driver-cassandra.c
index 13f46f45726..05cd3d97592 100644
--- a/src/lib-sql/driver-cassandra.c
+++ b/src/lib-sql/driver-cassandra.c
@@ -8,10 +8,12 @@
 #include "net.h"
 #include "write-full.h"
 #include "time-util.h"
+#include "var-expand.h"
 #include "settings-parser.h"
 #include "sql-api-private.h"
 
 #ifdef BUILD_CASSANDRA
+#include <fcntl.h>
 #include <unistd.h>
 #include <cassandra.h>
 
@@ -66,6 +68,9 @@ struct cassandra_db {
 	ARRAY(struct cassandra_result *) results;
 	unsigned int callback_ids;
 
+	char *metrics_path;
+	struct timeout *to_metrics;
+
 	struct timeval first_fallback_sent[CASSANDRA_QUERY_TYPE_COUNT];
 	time_t last_fallback_warning[CASSANDRA_QUERY_TYPE_COUNT];
 	unsigned int fallback_failures[CASSANDRA_QUERY_TYPE_COUNT];
@@ -442,6 +447,9 @@ static void driver_cassandra_parse_connect_string(struct cassandra_db *db,
 		} else if (strcmp(key, "request_timeout") == 0) {
 			if (settings_get_time(value, &db->request_timeout_secs, &error) < 0)
 				i_fatal("cassandra: Invalid request_timeout '%s': %s", value, error);
+		} else if (strcmp(key, "metrics") == 0) {
+			i_free(db->metrics_path);
+			db->metrics_path = i_strdup(value);
 		} else {
 			i_fatal("cassandra: Unknown connect string: %s", key);
 		}
@@ -461,6 +469,69 @@ static void driver_cassandra_parse_connect_string(struct cassandra_db *db,
 	db->hosts = i_strdup(str_c(hosts));
 }
 
+static void
+driver_cassandra_get_metrics_json(struct cassandra_db *db, string_t *dest)
+{
+#define ADD_UINT64(_struct, _field) \
+	str_printfa(dest, "\""#_field"\": %llu,", (unsigned long long)metrics._struct._field);
+#define ADD_DOUBLE(_struct, _field) \
+	str_printfa(dest, "\""#_field"\": %02lf,", metrics._struct._field);
+	CassMetrics metrics;
+
+	cass_session_get_metrics(db->session, &metrics);
+	str_append(dest, "{ \"requests\": {");
+	ADD_UINT64(requests, min);
+	ADD_UINT64(requests, max);
+	ADD_UINT64(requests, mean);
+	ADD_UINT64(requests, stddev);
+	ADD_UINT64(requests, median);
+	ADD_UINT64(requests, percentile_75th);
+	ADD_UINT64(requests, percentile_95th);
+	ADD_UINT64(requests, percentile_98th);
+	ADD_UINT64(requests, percentile_99th);
+	ADD_UINT64(requests, percentile_999th);
+	ADD_DOUBLE(requests, mean_rate);
+	ADD_DOUBLE(requests, one_minute_rate);
+	ADD_DOUBLE(requests, five_minute_rate);
+	ADD_DOUBLE(requests, fifteen_minute_rate);
+	str_truncate(dest, str_len(dest)-1);
+	str_append(dest, "}, \"stats\": {");
+	ADD_UINT64(stats, total_connections);
+	ADD_UINT64(stats, available_connections);
+	ADD_UINT64(stats, exceeded_pending_requests_water_mark);
+	ADD_UINT64(stats, exceeded_write_bytes_water_mark);
+	str_truncate(dest, str_len(dest)-1);
+	str_append(dest, "}, \"errors\": {");
+	ADD_UINT64(errors, connection_timeouts);
+	ADD_UINT64(errors, pending_request_timeouts);
+	ADD_UINT64(errors, request_timeouts);
+	str_truncate(dest, str_len(dest)-1);
+	str_append(dest, "}}");
+}
+
+static void driver_cassandra_metrics_write(struct cassandra_db *db)
+{
+	struct var_expand_table tab[] = {
+		{ '\0', NULL, NULL }
+	};
+	string_t *path = t_str_new(64);
+	string_t *data;
+	int fd;
+
+	var_expand(path, db->metrics_path, tab);
+
+	fd = open(str_c(path), O_WRONLY | O_CREAT | O_TRUNC | O_NONBLOCK, 0600);
+	if (fd == -1) {
+		i_error("creat(%s) failed: %m", str_c(path));
+		return;
+	}
+	data = t_str_new(1024);
+	driver_cassandra_get_metrics_json(db, data);
+	if (write_full(fd, str_data(data), str_len(data)) < 0)
+		i_error("write(%s) failed: %m", str_c(path));
+	i_close_fd(&fd);
+}
+
 static struct sql_db *driver_cassandra_init_v(const char *connect_string)
 {
 	struct cassandra_db *db;
@@ -489,6 +560,8 @@ static struct sql_db *driver_cassandra_init_v(const char *connect_string)
 	if (db->num_threads != 0)
 		cass_cluster_set_num_threads_io(db->cluster, db->num_threads);
 	db->session = cass_session_new();
+	if (db->metrics_path != NULL)
+		db->to_metrics = timeout_add(1000, driver_cassandra_metrics_write, db);
 	i_array_init(&db->results, 16);
 	i_array_init(&db->callbacks, 16);
 	return &db->api;
@@ -508,6 +581,9 @@ static void driver_cassandra_deinit_v(struct sql_db *_db)
 	cass_session_free(db->session);
 	cass_cluster_free(db->cluster);
 	cass_timestamp_gen_free(db->timestamp_gen);
+	if (db->to_metrics != NULL)
+		timeout_remove(&db->to_metrics);
+	i_free(db->metrics_path);
 	i_free(db->hosts);
 	i_free(db->error);
 	i_free(db->keyspace);

From a436e90ca79d52cc7dedf82ebb96286af6f2be87 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 26 May 2016 18:25:06 +0300
Subject: [PATCH 175/450] dict-sql: Fixed memory leak

---
 src/lib-dict/dict-sql-settings.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/lib-dict/dict-sql-settings.c b/src/lib-dict/dict-sql-settings.c
index 5b090061edd..25e08bbf55e 100644
--- a/src/lib-dict/dict-sql-settings.c
+++ b/src/lib-dict/dict-sql-settings.c
@@ -248,7 +248,7 @@ dict_sql_settings_read(const char *path, const char **error_r)
 {
 	struct setting_parser_ctx ctx;
 	struct dict_sql_settings_cache *cache;
-	pool_t pool = pool_alloconly_create("dict sql settings", 1024);
+	pool_t pool;
 
 	if (!hash_table_is_created(dict_sql_settings_cache)) {
 		hash_table_create(&dict_sql_settings_cache, default_pool, 0,
@@ -260,6 +260,7 @@ dict_sql_settings_read(const char *path, const char **error_r)
 		return cache->set;
 
 	memset(&ctx, 0, sizeof(ctx));
+	pool = pool_alloconly_create("dict sql settings", 1024);
 	ctx.pool = pool;
 	ctx.set = p_new(pool, struct dict_sql_settings, 1);
 	t_array_init(&ctx.cur_fields, 16);

From d1843ca668fe692891ce49b47c362d49c949a002 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 26 May 2016 21:40:30 +0300
Subject: [PATCH 176/450] sdbox: Make sure we don't crash when altmove-flag is
 set but alt path is missing.

---
 src/lib-storage/index/dbox-single/sdbox-file.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/lib-storage/index/dbox-single/sdbox-file.c b/src/lib-storage/index/dbox-single/sdbox-file.c
index 4cdaaf2aa79..347d77f84d8 100644
--- a/src/lib-storage/index/dbox-single/sdbox-file.c
+++ b/src/lib-storage/index/dbox-single/sdbox-file.c
@@ -302,6 +302,8 @@ int sdbox_file_move(struct dbox_file *file, bool alt_path)
 
 	if (dbox_file_is_in_alt(file) == alt_path)
 		return 0;
+	if (file->alt_path == NULL)
+		return 0;
 
 	if (stat(file->cur_path, &st) < 0 && errno == ENOENT) {
 		/* already expunged/moved by another session */

From 8baf2344f9e2b98aaac46eb3830bcca218cacc3d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 26 May 2016 19:13:40 +0300
Subject: [PATCH 177/450] doveadm: Fixed sending command -parameters to
 doveadm-server

---
 src/doveadm/doveadm-mail.c | 44 +++++++++++++++++++++++---------------
 1 file changed, 27 insertions(+), 17 deletions(-)

diff --git a/src/doveadm/doveadm-mail.c b/src/doveadm/doveadm-mail.c
index 39faca2d98f..ca89a94c8cf 100644
--- a/src/doveadm/doveadm-mail.c
+++ b/src/doveadm/doveadm-mail.c
@@ -924,11 +924,9 @@ void
 doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx)
 {
 	struct doveadm_mail_cmd_context *mctx;
-	const char *wildcard_user, *username_args[3] = { NULL, NULL, NULL };
+	const char *wildcard_user;
 	const char *fieldstr;
-	unsigned int username_args_count;
-
-	ARRAY_TYPE(const_string) pargv;
+	ARRAY_TYPE(const_string) pargv, full_args;
 	int i;
 	struct doveadm_mail_cmd mail_cmd = {
 		cctx->cmd->mail_cmd, cctx->cmd->name, cctx->cmd->usage
@@ -944,6 +942,7 @@ doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx)
 	mctx->cur_username = cctx->username;
 	mctx->iterate_all_users = FALSE;
 	wildcard_user = NULL;
+	p_array_init(&full_args, mctx->pool, 8);
 	p_array_init(&pargv, mctx->pool, 8);
 
 	for(i=0;i<cctx->argc;i++) {
@@ -954,7 +953,8 @@ doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx)
 
 		if (strcmp(arg->name, "all-users") == 0) {
 			mctx->iterate_all_users = arg->value.v_bool;
-			username_args[0] = "-A";
+			fieldstr = "-A";
+			array_append(&full_args, &fieldstr, 1);
 		} else if (strcmp(arg->name, "socket-path") == 0) {
 			doveadm_settings->doveadm_socket_path = arg->value.v_string;
 			if (doveadm_settings->doveadm_worker_count == 0)
@@ -962,8 +962,10 @@ doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx)
 		} else if (strcmp(arg->name, "user") == 0) {
 			mctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP;
 			mctx->cur_username = arg->value.v_string;
-			username_args[0] = "-u";
-			username_args[1] = arg->value.v_string;
+
+			fieldstr = "-u";
+			array_append(&full_args, &fieldstr, 1);
+			array_append(&full_args, &arg->value.v_string, 1);
 			if (strchr(mctx->cur_username, '*') != NULL ||
 			    strchr(mctx->cur_username, '?') != NULL) {
 				wildcard_user = mctx->cur_username;
@@ -975,8 +977,10 @@ doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx)
 			mctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP;
 			wildcard_user = "*";
 			mctx->users_list_input = arg->value.v_istream;
-			username_args[0] = "-F";
-			username_args[1] = ""; /* value doesn't really matter */
+			fieldstr = "-F";
+			array_append(&full_args, &fieldstr, 1);
+			fieldstr = ""; /* value doesn't really matter */
+			array_append(&full_args, &fieldstr, 1);
 			i_stream_ref(mctx->users_list_input);
 		} else if (strcmp(arg->name, "field") == 0 ||
 			   strcmp(arg->name, "flag") == 0) {
@@ -1001,8 +1005,15 @@ doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx)
 		/* Keep all named special parameters above this line */
 
 		} else if (mctx->v.parse_arg != NULL && arg->short_opt != '\0') {
+			const char *short_opt_str = p_strdup_printf(
+				mctx->pool, "-%c", arg->short_opt);
+
 			optarg = (char*)arg->value.v_string;
 			mctx->v.parse_arg(mctx, arg->short_opt);
+
+			array_append(&full_args, &short_opt_str, 1);
+			if (arg->type == CMD_PARAM_STR)
+				array_append(&full_args, &arg->value.v_string, 1);
 		} else if ((arg->flags & CMD_PARAM_FLAG_POSITIONAL) != 0) {
 			/* feed this into pargv */
 			if (arg->type == CMD_PARAM_ARRAY)
@@ -1018,14 +1029,13 @@ doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx)
 	}
 
 	array_append_zero(&pargv);
-	/* -A, -u and -F parameters need to be included in full_args so that
-	   they're sent to doveadm-server. This is needed so that
-	   doveadm-server returns the username header when needed. */
-	username_args_count = str_array_length(username_args);
-	if (username_args_count > 0)
-		array_insert(&pargv, 0, username_args, username_args_count);
-	mctx->args = array_idx(&pargv, username_args_count);
-	mctx->full_args = array_idx(&pargv, 0);
+	/* All the -parameters need to be included in full_args so that
+	   they're sent to doveadm-server. */
+	unsigned int args_pos = array_count(&full_args);
+	array_append_array(&full_args, &pargv);
+
+	mctx->args = array_idx(&full_args, args_pos);
+	mctx->full_args = array_idx(&full_args, 0);
 	mctx->cli = cctx->cli;
 
 	doveadm_mail_cmd_exec(mctx, cctx, wildcard_user);

From f71e3eb2783324207272a41f4b6022c5afa6f496 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 27 May 2016 14:33:47 +0300
Subject: [PATCH 178/450] lib-fs: fs-metawrap: Improved error logging on
 corrupted files.

---
 src/lib-fs/istream-metawrap.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/lib-fs/istream-metawrap.c b/src/lib-fs/istream-metawrap.c
index d757f51a43b..55609c5512a 100644
--- a/src/lib-fs/istream-metawrap.c
+++ b/src/lib-fs/istream-metawrap.c
@@ -25,7 +25,8 @@ static int metadata_header_read(struct metawrap_istream *mstream)
 		p = strchr(line, ':');
 		if (p == NULL) {
 			io_stream_set_error(&mstream->istream.iostream,
-				"Metadata header line is missing ':'");
+				"Metadata header line is missing ':' at offset %"PRIuUOFF_T,
+				mstream->istream.v_offset);
 			mstream->istream.istream.stream_errno = EINVAL;
 			return -1;
 		}
@@ -38,7 +39,8 @@ static int metadata_header_read(struct metawrap_istream *mstream)
 				mstream->istream.parent->stream_errno;
 		} else {
 			io_stream_set_error(&mstream->istream.iostream,
-				"Metadata header is missing ending line");
+				"Metadata header is missing ending line at offset %"PRIuUOFF_T,
+				mstream->istream.v_offset);
 			mstream->istream.istream.stream_errno = EINVAL;
 			return -1;
 		}

From ec6e0273964197adb3f136c01b5ab456f1ea1ebf Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 27 May 2016 14:38:17 +0300
Subject: [PATCH 179/450] lib-fs: Fix to previous fs-metawrap commit

Argh. I was sure I compiled this before pushing it.
---
 src/lib-fs/istream-metawrap.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib-fs/istream-metawrap.c b/src/lib-fs/istream-metawrap.c
index 55609c5512a..edba22f124d 100644
--- a/src/lib-fs/istream-metawrap.c
+++ b/src/lib-fs/istream-metawrap.c
@@ -26,7 +26,7 @@ static int metadata_header_read(struct metawrap_istream *mstream)
 		if (p == NULL) {
 			io_stream_set_error(&mstream->istream.iostream,
 				"Metadata header line is missing ':' at offset %"PRIuUOFF_T,
-				mstream->istream.v_offset);
+				mstream->istream.istream.v_offset);
 			mstream->istream.istream.stream_errno = EINVAL;
 			return -1;
 		}
@@ -40,7 +40,7 @@ static int metadata_header_read(struct metawrap_istream *mstream)
 		} else {
 			io_stream_set_error(&mstream->istream.iostream,
 				"Metadata header is missing ending line at offset %"PRIuUOFF_T,
-				mstream->istream.v_offset);
+				mstream->istream.istream.v_offset);
 			mstream->istream.istream.stream_errno = EINVAL;
 			return -1;
 		}

From e56b1f272cbc5ea0d3cf1729e54f733e11999ea7 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 27 May 2016 18:30:01 +0300
Subject: [PATCH 180/450] dsync: Debug logging improvement.

---
 src/doveadm/dsync/dsync-mailbox-import.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c
index 6a44f88a464..6b5c6bf7554 100644
--- a/src/doveadm/dsync/dsync-mailbox-import.c
+++ b/src/doveadm/dsync/dsync-mailbox-import.c
@@ -124,6 +124,10 @@ struct dsync_mailbox_importer {
 	unsigned int delete_mailbox:1;
 };
 
+static const char *dsync_mail_change_type_names[] = {
+	"save", "expunge", "flag-change"
+};
+
 static bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer,
 					const struct dsync_mail *mail,
 					struct importer_new_mail *all_newmails,
@@ -1710,7 +1714,8 @@ int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,
 		result = "New mail";
 	}
 
-	imp_debug(importer, "Import change GUID=%s UID=%u hdr_hash=%s result=%s",
+	imp_debug(importer, "Import change type=%s GUID=%s UID=%u hdr_hash=%s result=%s",
+		  dsync_mail_change_type_names[change->type],
 		  change->guid != NULL ? change->guid : "<unknown>", change->uid,
 		  change->hdr_hash != NULL ? change->hdr_hash : "", result);
 

From 2896dfdda08160527d881e20118952c082821641 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 27 May 2016 21:32:23 +0300
Subject: [PATCH 181/450] lazy-expunge: Fixed error logging.

---
 .../lazy-expunge/lazy-expunge-plugin.c        | 46 ++++++++++++++-----
 1 file changed, 35 insertions(+), 11 deletions(-)

diff --git a/src/plugins/lazy-expunge/lazy-expunge-plugin.c b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
index e69e372bcb8..580a8b00262 100644
--- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c
+++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
@@ -52,7 +52,9 @@ struct lazy_expunge_transaction {
 	pool_t pool;
 	HASH_TABLE(const char *, void *) guids;
 
-	bool failed;
+	char *delayed_errstr;
+	enum mail_error delayed_error;
+
 	bool copy_only_last_instance;
 };
 
@@ -240,6 +242,19 @@ static bool lazy_expunge_is_internal_mailbox(struct mailbox *box)
 	return FALSE;
 }
 
+static void lazy_expunge_set_error(struct lazy_expunge_transaction *lt,
+				   struct mail_storage *storage)
+{
+	const char *errstr;
+	enum mail_error error;
+
+	if (lt->delayed_error != MAIL_ERROR_NONE)
+		return;
+	errstr = mail_storage_get_last_error(storage, &error);
+	lt->delayed_error = error;
+	lt->delayed_errstr = i_strdup(errstr);
+}
+
 static void lazy_expunge_mail_expunge(struct mail *_mail)
 {
 	struct mail_namespace *ns = _mail->box->list->ns;
@@ -257,7 +272,7 @@ static void lazy_expunge_mail_expunge(struct mail *_mail)
 	/* don't copy the mail if we're expunging from lazy_expunge
 	   namespace (even if it's via a virtual mailbox) */
 	if (mail_get_backend_mail(_mail, &real_mail) < 0) {
-		lt->failed = TRUE;
+		lazy_expunge_set_error(lt, _mail->box->storage);
 		return;
 	}
 	if (lazy_expunge_is_internal_mailbox(real_mail->box)) {
@@ -270,7 +285,7 @@ static void lazy_expunge_mail_expunge(struct mail *_mail)
 		   lazy_expunge namespace. other instances will be expunged
 		   immediately. */
 		if ((ret = lazy_expunge_mail_is_last_instace(_mail)) < 0) {
-			lt->failed = TRUE;
+			lazy_expunge_set_error(lt, _mail->box->storage);
 			return;
 		}
 		if (ret == 0) {
@@ -286,14 +301,14 @@ static void lazy_expunge_mail_expunge(struct mail *_mail)
 			mail_storage_set_critical(_mail->box->storage,
 				"lazy_expunge: Couldn't open expunge mailbox: "
 				"%s", error);
-			lt->failed = TRUE;
+			lazy_expunge_set_error(lt, _mail->box->storage);
 			return;
 		}
 		if (mailbox_sync(lt->dest_box, 0) < 0) {
 			mail_storage_set_critical(_mail->box->storage,
 				"lazy_expunge: Couldn't sync expunge mailbox");
+			lazy_expunge_set_error(lt, lt->dest_box->storage);
 			mailbox_free(&lt->dest_box);
-			lt->failed = TRUE;
 			return;
 		}
 
@@ -305,7 +320,7 @@ static void lazy_expunge_mail_expunge(struct mail *_mail)
 	mailbox_save_copy_flags(save_ctx, _mail);
 	save_ctx->data.flags &= ~MAIL_DELETED;
 	if (mailbox_copy(&save_ctx, _mail) < 0 && !_mail->expunged)
-		lt->failed = TRUE;
+		lazy_expunge_set_error(lt, lt->dest_box->storage);
 	mmail->super.expunge(_mail);
 }
 
@@ -337,6 +352,7 @@ static void lazy_expunge_transaction_free(struct lazy_expunge_transaction *lt)
 		hash_table_destroy(&lt->guids);
 	if (lt->pool != NULL)
 		pool_unref(&lt->pool);
+	i_free(lt->delayed_errstr);
 	i_free(lt);
 }
 
@@ -348,16 +364,24 @@ lazy_expunge_transaction_commit(struct mailbox_transaction_context *ctx,
 	struct lazy_expunge_transaction *lt = LAZY_EXPUNGE_CONTEXT(ctx);
 	int ret;
 
-	if (lt->dest_trans != NULL && !lt->failed) {
-		if (mailbox_transaction_commit(&lt->dest_trans) < 0)
-			lt->failed = TRUE;
+	if (lt->dest_trans != NULL && lt->delayed_error == MAIL_ERROR_NONE) {
+		if (mailbox_transaction_commit(&lt->dest_trans) < 0) {
+			lazy_expunge_set_error(lt, lt->dest_trans->box->storage);
+		}
 	}
 
-	if (lt->failed) {
+	if (lt->delayed_error == MAIL_ERROR_NONE)
+		ret = mbox->super.transaction_commit(ctx, changes_r);
+	else if (lt->delayed_error != MAIL_ERROR_TEMP) {
+		mail_storage_set_error(ctx->box->storage, lt->delayed_error,
+				       lt->delayed_errstr);
 		mbox->super.transaction_rollback(ctx);
 		ret = -1;
 	} else {
-		ret = mbox->super.transaction_commit(ctx, changes_r);
+		mail_storage_set_critical(ctx->box->storage,
+			"Lazy-expunge transaction failed: %s", lt->delayed_errstr);
+		mbox->super.transaction_rollback(ctx);
+		ret = -1;
 	}
 	lazy_expunge_transaction_free(lt);
 	return ret;

From 137dcd83227d4d7e82c537fafbe8d457160a112f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 27 May 2016 21:34:26 +0300
Subject: [PATCH 182/450] lazy-expunge: Don't fail expunge if mail was already
 expunged.

---
 src/plugins/lazy-expunge/lazy-expunge-plugin.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/plugins/lazy-expunge/lazy-expunge-plugin.c b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
index 580a8b00262..b9faf182e55 100644
--- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c
+++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
@@ -248,9 +248,15 @@ static void lazy_expunge_set_error(struct lazy_expunge_transaction *lt,
 	const char *errstr;
 	enum mail_error error;
 
+	errstr = mail_storage_get_last_error(storage, &error);
+	if (error == MAIL_ERROR_EXPUNGED) {
+		/* expunging failed because the mail was already expunged.
+		   we don't want to fail because of that. */
+		return;
+	}
+
 	if (lt->delayed_error != MAIL_ERROR_NONE)
 		return;
-	errstr = mail_storage_get_last_error(storage, &error);
 	lt->delayed_error = error;
 	lt->delayed_errstr = i_strdup(errstr);
 }

From 336eedf3be4d1f46dd64d4c9a32a27d73509a8c4 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 27 May 2016 21:55:56 +0300
Subject: [PATCH 183/450] dbox: Revert cache changes if some save/copy fails
 but transaction is committed.

---
 src/lib-storage/index/dbox-multi/mdbox-save.c  | 1 +
 src/lib-storage/index/dbox-single/sdbox-save.c | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/lib-storage/index/dbox-multi/mdbox-save.c b/src/lib-storage/index/dbox-multi/mdbox-save.c
index 6d1b354f770..857ff7b7b2f 100644
--- a/src/lib-storage/index/dbox-multi/mdbox-save.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-save.c
@@ -216,6 +216,7 @@ static int mdbox_save_finish_write(struct mail_save_context *_ctx)
 
 	if (ctx->ctx.failed) {
 		mail_index_expunge(ctx->ctx.trans, ctx->ctx.seq);
+		mail_cache_transaction_reset(ctx->ctx.ctx.transaction->cache_trans);
 		mdbox_map_append_abort(ctx->append_ctx);
 		array_delete(&ctx->mails, array_count(&ctx->mails) - 1, 1);
 		return -1;
diff --git a/src/lib-storage/index/dbox-single/sdbox-save.c b/src/lib-storage/index/dbox-single/sdbox-save.c
index 919a628225a..9f0eff8852b 100644
--- a/src/lib-storage/index/dbox-single/sdbox-save.c
+++ b/src/lib-storage/index/dbox-single/sdbox-save.c
@@ -195,6 +195,7 @@ static int dbox_save_finish_write(struct mail_save_context *_ctx)
 
 	if (ctx->ctx.failed) {
 		mail_index_expunge(ctx->ctx.trans, ctx->ctx.seq);
+		mail_cache_transaction_reset(ctx->ctx.ctx.transaction->cache_trans);
 		dbox_file_append_rollback(&ctx->append_ctx);
 		dbox_file_unlink(*files);
 		dbox_file_unref(files);

From e7f2fde236b5d43887ba703ea34ebe7678658061 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Wed, 24 Feb 2016 12:10:19 +0200
Subject: [PATCH 184/450] doveadm-director: Convert to ver2 infra

---
 src/doveadm/doveadm-director.c | 561 ++++++++++++++++++++-------------
 1 file changed, 339 insertions(+), 222 deletions(-)

diff --git a/src/doveadm/doveadm-director.c b/src/doveadm/doveadm-director.c
index 76accd25661..516cafabecb 100644
--- a/src/doveadm/doveadm-director.c
+++ b/src/doveadm/doveadm-director.c
@@ -21,6 +21,13 @@ struct director_context {
 	const char *socket_path;
 	const char *users_path;
 	const char *tag;
+	const char *user;
+	const char *host;
+	const char *ip;
+	const char *port;
+	const char *vhost_count;
+
+	struct istream *users_input;
 	struct istream *input;
 	bool explicit_socket_path;
 	bool hash_map, user_map, force_flush;
@@ -33,10 +40,9 @@ struct user_list {
 
 HASH_TABLE_DEFINE_TYPE(user_list, void *, struct user_list *);
 
-extern struct doveadm_cmd doveadm_cmd_director[];
-
-static void director_cmd_help(doveadm_command_t *cmd) ATTR_NORETURN;
-
+static void director_cmd_help(const struct doveadm_cmd_ver2 *cmd ATTR_UNUSED);
+static int director_get_host(const char *host, struct ip_addr **ips_r,
+                              unsigned int *ips_count_r) ATTR_WARN_UNUSED_RESULT;
 static void
 director_send(struct director_context *ctx, const char *data)
 {
@@ -86,55 +92,49 @@ static void director_disconnect(struct director_context *ctx)
 }
 
 static struct director_context *
-cmd_director_init(int argc, char *argv[], const char *getopt_args,
-		  doveadm_command_t *cmd)
+cmd_director_init(int argc, const struct doveadm_cmd_param *argv)
 {
 	struct director_context *ctx;
-	int c;
-
 	ctx = t_new(struct director_context, 1);
-	ctx->socket_path = t_strconcat(doveadm_settings->base_dir,
-				       "/director-admin", NULL);
-
-	while ((c = getopt(argc, argv, getopt_args)) > 0) {
-		switch (c) {
-		case 'a':
-			ctx->socket_path = optarg;
-			ctx->explicit_socket_path = TRUE;
-			break;
-		case 'f':
-			ctx->users_path = optarg;
-			break;
-		case 'F':
-			ctx->force_flush = TRUE;
-			break;
-		case 'h':
-			ctx->hash_map = TRUE;
-			break;
-		case 'u':
-			ctx->user_map = TRUE;
-			break;
-		case 't':
-			ctx->tag = optarg;
-			break;
-		default:
-			director_cmd_help(cmd);
-		}
-	}
+	if (!doveadm_cmd_param_str(argc, argv, "socket-path", &(ctx->socket_path)))
+		ctx->socket_path = t_strconcat(doveadm_settings->base_dir,
+					"/director-admin", NULL);
+	else
+		ctx->explicit_socket_path = TRUE;
+	if (!doveadm_cmd_param_bool(argc, argv, "user-map", &(ctx->user_map)))
+		ctx->user_map = FALSE;
+	if (!doveadm_cmd_param_bool(argc, argv, "hash-map", &(ctx->hash_map)))
+		ctx->hash_map = FALSE;
+	if (!doveadm_cmd_param_bool(argc, argv, "force-flush", &(ctx->force_flush)))
+		ctx->force_flush = FALSE;
+	if (!doveadm_cmd_param_istream(argc, argv, "users-file", &(ctx->users_input)))
+		ctx->users_input = NULL;
+	if (!doveadm_cmd_param_str(argc, argv, "tag", &(ctx->tag)))
+		ctx->tag = NULL;
+	if (!doveadm_cmd_param_str(argc, argv, "user", &(ctx->user)))
+		ctx->user = NULL;
+	if (!doveadm_cmd_param_str(argc, argv, "host", &(ctx->host)))
+		ctx->host = NULL;
+	if (!doveadm_cmd_param_str(argc, argv, "ip", &(ctx->ip)))
+		ctx->ip = NULL;
+	if (!doveadm_cmd_param_str(argc, argv, "port", &(ctx->port)))
+		ctx->port = NULL;
+	if (!doveadm_cmd_param_str(argc, argv, "vhost-count", &(ctx->vhost_count)))
+		ctx->vhost_count = NULL;
 	if (!ctx->user_map)
 		director_connect(ctx);
 	return ctx;
 }
 
+
 static void
-cmd_director_status_user(struct director_context *ctx, char *argv[])
+cmd_director_status_user(struct director_context *ctx)
 {
-	const char *user = argv[0], *tag = argv[1];
 	const char *line, *const *args;
 	unsigned int expires;
 
-	director_send(ctx, t_strdup_printf("USER-LOOKUP\t%s\t%s\n", user,
-					   tag != NULL ? tag : ""));
+	director_send(ctx, t_strdup_printf("USER-LOOKUP\t%s\t%s\n", ctx->user,
+					   ctx->tag != NULL ? ctx->tag : ""));
 	line = i_stream_read_next_line(ctx->input);
 	if (line == NULL) {
 		i_error("Lookup failed");
@@ -150,25 +150,38 @@ cmd_director_status_user(struct director_context *ctx, char *argv[])
 		return;
 	}
 
+	doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED);
+
+	doveadm_print_header_simple("status");
+	doveadm_print_header_simple("expires");
+	doveadm_print_header_simple("hashed");
+	doveadm_print_header_simple("initial-config");
+
+	doveadm_print_formatted_set_format("Current: %{status} (expires %{expires})\n" \
+					   "Hashed: %{hashed}\n" \
+					   "Initial config: %{initial-config}\n");
+
 	if (args[0][0] != '\0') {
-		printf("Current: %s (expires %s)\n",
-		       args[0], unixdate2str(expires));
+		doveadm_print(args[0]);
+		doveadm_print(unixdate2str(expires));
 	} else {
-		printf("Current: not assigned\n");
+		doveadm_print("n/a");
+		doveadm_print("-1");
 	}
-	printf("Hashed: %s\n", args[2]);
-	printf("Initial config: %s\n", args[3]);
+	doveadm_print(args[2]);
+	doveadm_print(args[3]);
+
 	director_disconnect(ctx);
 }
 
-static void cmd_director_status(int argc, char *argv[])
+static void cmd_director_status(const struct doveadm_cmd_ver2 *cmd ATTR_UNUSED, int argc, const struct doveadm_cmd_param *argv)
 {
 	struct director_context *ctx;
 	const char *line, *const *args;
 
-	ctx = cmd_director_init(argc, argv, "a:t:", cmd_director_status);
-	if (argv[optind] != NULL) {
-		cmd_director_status_user(ctx, argv+optind);
+	ctx = cmd_director_init(argc, argv);
+	if (ctx->user != NULL) {
+		cmd_director_status_user(ctx);
 		return;
 	}
 
@@ -254,27 +267,20 @@ userdb_get_user_list(const char *auth_socket_path, pool_t pool,
 }
 
 static void
-user_file_get_user_list(const char *path, pool_t pool,
+user_file_get_user_list(struct istream *input, pool_t pool,
 			HASH_TABLE_TYPE(user_list) users)
 {
-	struct istream *input;
 	const char *username;
-	int fd;
 
-	fd = open(path, O_RDONLY);
-	if (fd == -1)
-		i_fatal("open(%s) failed: %m", path);
-	input = i_stream_create_fd_autoclose(&fd, (size_t)-1);
 	while ((username = i_stream_read_next_line(input)) != NULL)
 		user_list_add(username, pool, users);
-	i_stream_unref(&input);
 }
 
-static void director_get_host(const char *host, struct ip_addr **ips_r,
+static int director_get_host(const char *host, struct ip_addr **ips_r,
 			      unsigned int *ips_count_r)
 {
 	struct ip_addr ip;
-	int ret;
+	int ret = 0;
 
 	if (net_addr2ip(host, &ip) == 0) {
 		*ips_r = t_new(struct ip_addr, 1);
@@ -283,10 +289,14 @@ static void director_get_host(const char *host, struct ip_addr **ips_r,
 	} else {
 		ret = net_gethostbyname(host, ips_r, ips_count_r);
 		if (ret != 0) {
-			i_fatal("gethostname(%s) failed: %s", host,
+			i_error("gethostname(%s) failed: %s", host,
 				net_gethosterror(ret));
+			doveadm_exit_code = EX_TEMPFAIL;
+			return ret;
 		}
 	}
+
+	return ret;
 }
 
 static bool ip_find(const struct ip_addr *ips, unsigned int ips_count,
@@ -301,7 +311,7 @@ static bool ip_find(const struct ip_addr *ips, unsigned int ips_count,
 	return FALSE;
 }
 
-static void cmd_director_map(int argc, char *argv[])
+static void cmd_director_map(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
 {
 	struct director_context *ctx;
 	const char *line, *const *args;
@@ -311,17 +321,16 @@ static void cmd_director_map(int argc, char *argv[])
 	struct user_list *user;
 	unsigned int ips_count, user_hash, expires;
 
-	ctx = cmd_director_init(argc, argv, "a:f:hu", cmd_director_map);
-	argc -= optind;
-	argv += optind;
-	if (argc > 1 ||
-	    (ctx->hash_map && ctx->user_map) ||
-	    ((ctx->hash_map || ctx->user_map) && argc == 0))
-		director_cmd_help(cmd_director_map);
+	ctx = cmd_director_init(argc, argv);
+
+	if ((ctx->hash_map && ctx->user_map) && ctx->host == NULL) {
+		director_cmd_help(cmd);
+	return;
+	}
 
 	if (ctx->user_map) {
 		/* user -> hash mapping */
-		user_hash = mail_user_hash(argv[0], doveadm_settings->director_username_hash);
+		user_hash = mail_user_hash(ctx->host, doveadm_settings->director_username_hash);
 		doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
 		doveadm_print_header("hash", "hash", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
 		doveadm_print(t_strdup_printf("%u", user_hash));
@@ -329,22 +338,22 @@ static void cmd_director_map(int argc, char *argv[])
 		return;
 	}
 
-	if (argv[0] == NULL || ctx->hash_map)
+	if (ctx->host == NULL || ctx->hash_map)
 		ips_count = 0;
 	else
-		director_get_host(argv[0], &ips, &ips_count);
+		if (director_get_host(ctx->host, &ips, &ips_count) != 0) return;
 
 	pool = pool_alloconly_create("director map users", 1024*128);
 	hash_table_create_direct(&users, pool, 0);
-	if (ctx->users_path == NULL)
+	if (ctx->users_input == NULL)
 		userdb_get_user_list(NULL, pool, users);
 	else
-		user_file_get_user_list(ctx->users_path, pool, users);
+		user_file_get_user_list(ctx->users_input, pool, users);
 
 	if (ctx->hash_map) {
 		/* hash -> usernames mapping */
-		if (str_to_uint(argv[0], &user_hash) < 0)
-			i_fatal("Invalid username hash: %s", argv[0]);
+		if (str_to_uint(ctx->host, &user_hash) < 0)
+			i_fatal("Invalid username hash: %s", ctx->host);
 
 		doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
 		doveadm_print_header("user", "user", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
@@ -407,37 +416,39 @@ static void cmd_director_map(int argc, char *argv[])
 }
 
 static void
-cmd_director_add_or_update(int argc, char *argv[], doveadm_command_t *cmd_func,
-			   bool update)
+cmd_director_add_or_update(const struct doveadm_cmd_ver2 *dcmd, int argc, const struct doveadm_cmd_param *argv, bool update)
 {
 	const char *director_cmd = update ? "HOST-UPDATE" : "HOST-SET";
 	struct director_context *ctx;
 	struct ip_addr *ips;
 	unsigned int i, ips_count, vhost_count = UINT_MAX;
-	const char *host, *line;
+	const char *line, *host;
 	string_t *cmd;
 
-	ctx = cmd_director_init(argc, argv, update ? "a:" : "a:t:", cmd_func);
+	ctx = cmd_director_init(argc, argv);
 	if (ctx->tag != NULL && ctx->tag[0] == '\0')
 		ctx->tag = NULL;
-	host = argv[optind++];
-	if (host == NULL)
-		director_cmd_help(cmd_func);
-	if (argv[optind] != NULL) {
-		if (str_to_uint(argv[optind++], &vhost_count) < 0)
-			director_cmd_help(cmd_func);
-	} else if (strcmp(director_cmd, "HOST-UPDATE") == 0)
-		director_cmd_help(cmd_func);
-
-	if (argv[optind] != NULL)
-		director_cmd_help(cmd_func);
+	if (ctx->host == NULL) {
+		director_cmd_help(dcmd);
+		return;
+	}
+	if (ctx->vhost_count != NULL) {
+		if (str_to_uint(ctx->vhost_count, &vhost_count) < 0) {
+			director_cmd_help(dcmd);
+			return;
+		}
+	} else if (update) {
+		director_cmd_help(dcmd);
+		return;
+	}
 
+	host = ctx->host;
 	if (ctx->tag == NULL) {
-		ctx->tag = strchr(host, '@');
+		ctx->tag = strchr(ctx->host, '@');
 		if (ctx->tag != NULL)
-			host = t_strdup_until(host, ctx->tag++);
+			host = t_strdup_until(ctx->host, ctx->tag++);
 	}
-	director_get_host(host, &ips, &ips_count);
+	if (director_get_host(host, &ips, &ips_count) != 0) return;
 	cmd = t_str_new(128);
 	for (i = 0; i < ips_count; i++) {
 		str_truncate(cmd, 0);
@@ -452,43 +463,45 @@ cmd_director_add_or_update(int argc, char *argv[], doveadm_command_t *cmd_func,
 	for (i = 0; i < ips_count; i++) {
 		line = i_stream_read_next_line(ctx->input);
 		if (line == NULL || strcmp(line, "OK") != 0) {
-			fprintf(stderr, "%s: %s\n", net_ip2addr(&ips[i]),
+			i_error("%s: %s\n", net_ip2addr(&ips[i]),
 				line == NULL ? "failed" :
 				strcmp(line, "NOTFOUND") == 0 ?
 				"doesn't exist" : line);
 			doveadm_exit_code = EX_TEMPFAIL;
 		} else if (doveadm_verbose) {
-			printf("%s: OK\n", net_ip2addr(&ips[i]));
+			i_info("%s: OK\n", net_ip2addr(&ips[i]));
 		}
 	}
 	director_disconnect(ctx);
 }
 
-static void cmd_director_add(int argc, char *argv[])
+static void cmd_director_add(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
 {
-	cmd_director_add_or_update(argc, argv, cmd_director_add, FALSE);
+	cmd_director_add_or_update(cmd, argc, argv, FALSE);
 }
 
-static void cmd_director_update(int argc, char *argv[])
+static void cmd_director_update(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
 {
-	cmd_director_add_or_update(argc, argv, cmd_director_update, TRUE);
+	cmd_director_add_or_update(cmd, argc, argv, TRUE);
 }
 
 static void
-cmd_director_ipcmd(const char *cmd_name, doveadm_command_t *cmd,
-		   const char *success_result, int argc, char *argv[])
+cmd_director_ipcmd(const char *cmd_name, const struct doveadm_cmd_ver2 *cmd, const char *success_result,
+	int argc, const struct doveadm_cmd_param *argv)
 {
 	struct director_context *ctx;
 	struct ip_addr *ips;
 	unsigned int i, ips_count;
 	const char *host, *line;
 
-	ctx = cmd_director_init(argc, argv, "a:", cmd);
-	host = argv[optind++];
-	if (host == NULL || argv[optind] != NULL)
+	ctx = cmd_director_init(argc, argv);
+	host = ctx->host;
+	if (host == NULL) {
 		director_cmd_help(cmd);
+		return;
+	}
 
-	director_get_host(host, &ips, &ips_count);
+	if (director_get_host(host, &ips, &ips_count) != 0) return;
 	for (i = 0; i < ips_count; i++) {
 		director_send(ctx, t_strdup_printf(
 			"%s\t%s\n", cmd_name, net_ip2addr(&ips[i])));
@@ -496,55 +509,52 @@ cmd_director_ipcmd(const char *cmd_name, doveadm_command_t *cmd,
 	for (i = 0; i < ips_count; i++) {
 		line = i_stream_read_next_line(ctx->input);
 		if (line != NULL && strcmp(line, "NOTFOUND") == 0) {
-			fprintf(stderr, "%s: doesn't exist\n",
+			i_error("%s: doesn't exist\n",
 				net_ip2addr(&ips[i]));
 			if (doveadm_exit_code == 0)
 				doveadm_exit_code = DOVEADM_EX_NOTFOUND;
 		} else if (line == NULL || strcmp(line, "OK") != 0) {
-			fprintf(stderr, "%s: %s\n", net_ip2addr(&ips[i]),
+			i_error("%s: %s\n", net_ip2addr(&ips[i]),
 				line == NULL ? "failed" : line);
 			doveadm_exit_code = EX_TEMPFAIL;
 		} else if (doveadm_verbose) {
-			printf("%s: %s\n", net_ip2addr(&ips[i]), success_result);
+			i_info("%s: %s\n", net_ip2addr(&ips[i]), success_result);
 		}
 	}
 	director_disconnect(ctx);
 }
 
-static void cmd_director_remove(int argc, char *argv[])
+static void cmd_director_remove(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
 {
-	cmd_director_ipcmd("HOST-REMOVE", cmd_director_remove,
-			   "removed", argc, argv);
+	cmd_director_ipcmd("HOST-REMOVE", cmd, "removed", argc, argv);
 }
 
-static void cmd_director_up(int argc, char *argv[])
+static void cmd_director_up(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
 {
-	cmd_director_ipcmd("HOST-UP", cmd_director_up,
-			   "up", argc, argv);
+	cmd_director_ipcmd("HOST-UP", cmd, "up", argc, argv);
 }
 
-static void cmd_director_down(int argc, char *argv[])
+static void cmd_director_down(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
 {
-	cmd_director_ipcmd("HOST-DOWN", cmd_director_down,
-			   "down", argc, argv);
+	cmd_director_ipcmd("HOST-DOWN", cmd, "down", argc, argv);
 }
 
-static void cmd_director_move(int argc, char *argv[])
+static void cmd_director_move(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
 {
 	struct director_context *ctx;
 	struct ip_addr *ips;
 	unsigned int ips_count, user_hash;
-	const char *host, *line, *ip_str;
+	const char *line, *ip_str;
 
-	ctx = cmd_director_init(argc, argv, "a:", cmd_director_move);
-	if (argv[optind] == NULL || argv[optind+1] == NULL ||
-	    argv[optind+2] != NULL)
-		director_cmd_help(cmd_director_move);
+	ctx = cmd_director_init(argc, argv);
+	if (ctx->user == NULL || ctx->host == NULL) {
+		director_cmd_help(cmd);
+		return;
+	}
 
-	user_hash = mail_user_hash(argv[optind++], doveadm_settings->director_username_hash);
-	host = argv[optind];
+	user_hash = mail_user_hash(ctx->user, doveadm_settings->director_username_hash);
 
-	director_get_host(host, &ips, &ips_count);
+	if (director_get_host(ctx->host, &ips, &ips_count) != 0) return;
 	ip_str = net_ip2addr(&ips[0]);
 	director_send(ctx, t_strdup_printf(
 		"USER-MOVE\t%u\t%s\n", user_hash, ip_str));
@@ -554,7 +564,7 @@ static void cmd_director_move(int argc, char *argv[])
 		doveadm_exit_code = EX_TEMPFAIL;
 	} else if (strcmp(line, "OK") == 0) {
 		if (doveadm_verbose)
-			printf("User hash %u moved to %s\n", user_hash, ip_str);
+			i_info("User hash %u moved to %s\n", user_hash, ip_str);
 	} else if (strcmp(line, "NOTFOUND") == 0) {
 		i_error("Host '%s' doesn't exist", ip_str);
 		doveadm_exit_code = DOVEADM_EX_NOTFOUND;
@@ -569,25 +579,25 @@ static void cmd_director_move(int argc, char *argv[])
 	director_disconnect(ctx);
 }
 
-static void cmd_director_kick(int argc, char *argv[])
+static void cmd_director_kick(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
 {
 	struct director_context *ctx;
-	const char *username, *line;
-
-	ctx = cmd_director_init(argc, argv, "a:", cmd_director_kick);
-	if (argv[optind] == NULL || argv[optind+1] != NULL)
-		director_cmd_help(cmd_director_kick);
+	const char *line;
 
-	username = argv[optind];
+	ctx = cmd_director_init(argc, argv);
+	if (ctx->user == NULL) {
+		director_cmd_help(cmd);
+		return;
+	}
 
-	director_send(ctx, t_strdup_printf("USER-KICK\t%s\n", username));
+	director_send(ctx, t_strdup_printf("USER-KICK\t%s\n", ctx->user));
 	line = i_stream_read_next_line(ctx->input);
 	if (line == NULL) {
 		i_error("failed");
 		doveadm_exit_code = EX_TEMPFAIL;
 	} else if (strcmp(line, "OK") == 0) {
 		if (doveadm_verbose)
-			printf("User %s kicked\n", username);
+			i_info("User %s kicked", ctx->user);
 	} else {
 		i_error("failed: %s", line);
 		doveadm_exit_code = EX_TEMPFAIL;
@@ -610,37 +620,33 @@ static void cmd_director_flush_all(struct director_context *ctx)
 		i_error("failed: %s", line);
 		doveadm_exit_code = EX_TEMPFAIL;
 	} else if (doveadm_verbose)
-		printf("flushed\n");
+		i_info("flushed");
 	director_disconnect(ctx);
 }
 
-static void cmd_director_flush(int argc, char *argv[])
+static void cmd_director_flush(const struct doveadm_cmd_ver2 *cmd ATTR_UNUSED, int argc, const struct doveadm_cmd_param *argv)
 {
 	struct director_context *ctx;
 	struct ip_addr *ips;
 	unsigned int i, ips_count;
 	struct ip_addr ip;
-	const char *host, *line;
-	int ret;
+	const char *line;
 
-	ctx = cmd_director_init(argc, argv, "a:F", cmd_director_flush);
-	host = argv[optind++];
-	if (host == NULL || argv[optind] != NULL)
-		director_cmd_help(cmd_director_flush);
+	ctx = cmd_director_init(argc, argv);
+	if (ctx->host == NULL) {
+		director_cmd_help(cmd);
+		return;
+	}
 
-	if (strcmp(host, "all") == 0) {
+	if (strcmp(ctx->host, "all") == 0) {
 		cmd_director_flush_all(ctx);
 		return;
 	}
-	if (net_addr2ip(host, &ip) == 0) {
+	if (net_addr2ip(ctx->host, &ip) == 0) {
 		ips = &ip;
 		ips_count = 1;
 	} else {
-		ret = net_gethostbyname(host, &ips, &ips_count);
-		if (ret != 0) {
-			i_fatal("gethostname(%s) failed: %s", host,
-				net_gethosterror(ret));
-		}
+		if (director_get_host(ctx->host, &ips, &ips_count) != 0) return;
 	}
 
 	for (i = 0; i < ips_count; i++) {
@@ -651,42 +657,38 @@ static void cmd_director_flush(int argc, char *argv[])
 	for (i = 0; i < ips_count; i++) {
 		line = i_stream_read_next_line(ctx->input);
 		if (line != NULL && strcmp(line, "NOTFOUND") == 0) {
-			fprintf(stderr, "%s: doesn't exist\n",
+			i_warning("%s: doesn't exist",
 				net_ip2addr(&ips[i]));
 			if (doveadm_exit_code == 0)
 				doveadm_exit_code = DOVEADM_EX_NOTFOUND;
 		} else if (line == NULL || strcmp(line, "OK") != 0) {
-			fprintf(stderr, "%s: %s\n", net_ip2addr(&ips[i]),
+			i_warning("%s: %s", net_ip2addr(&ips[i]),
 				line == NULL ? "failed" : line);
 			doveadm_exit_code = EX_TEMPFAIL;
 		} else if (doveadm_verbose) {
-			printf("%s: flushed\n", net_ip2addr(&ips[i]));
+			i_info("%s: flushed", net_ip2addr(&ips[i]));
 		}
 	}
 	director_disconnect(ctx);
 }
 
-static void ATTR_FORMAT(3, 4)
-director_dump_cmd(struct director_context *ctx,
-		  const char *cmd, const char *args, ...)
-{
-	va_list va;
-
-	va_start(va, args);
-	printf("doveadm director %s ", cmd);
-	if (ctx->explicit_socket_path)
-		printf("-a %s ", ctx->socket_path);
-	vprintf(args, va);
-	putchar('\n');
-	va_end(va);
-}
-
-static void cmd_director_dump(int argc, char *argv[])
+static void cmd_director_dump(const struct doveadm_cmd_ver2 *cmd ATTR_UNUSED, int argc, const struct doveadm_cmd_param *argv)
 {
 	struct director_context *ctx;
 	const char *line, *const *args;
 
-	ctx = cmd_director_init(argc, argv, "a:", cmd_director_dump);
+	ctx = cmd_director_init(argc, argv);
+
+	doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED);
+	if (ctx->explicit_socket_path)
+		doveadm_print_formatted_set_format("doveadm director %{command} -a %{socket-path} %{host} %{vhost_count}");
+	else
+		doveadm_print_formatted_set_format("doveadm director %{command} %{host} %{vhost_count}");
+
+	doveadm_print_header_simple("command");
+	doveadm_print_header_simple("socket-path");
+	doveadm_print_header_simple("host");
+	doveadm_print_header_simple("vhost_count");
 
 	director_send(ctx, "HOST-LIST\n");
 	while ((line = i_stream_read_next_line(ctx->input)) != NULL) {
@@ -695,8 +697,10 @@ static void cmd_director_dump(int argc, char *argv[])
 		T_BEGIN {
 			args = t_strsplit_tab(line);
 			if (str_array_length(args) >= 2) {
-				director_dump_cmd(ctx, "add", "%s %s",
-						  args[0], args[1]);
+				doveadm_print("add");
+				doveadm_print(ctx->socket_path);
+				doveadm_print(args[0]);
+				doveadm_print(args[1]);
 			}
 		} T_END;
 	}
@@ -705,7 +709,10 @@ static void cmd_director_dump(int argc, char *argv[])
 	while ((line = i_stream_read_next_line(ctx->input)) != NULL) {
 		if (*line == '\0')
 			break;
-		director_dump_cmd(ctx, "remove", "%s", line);
+		doveadm_print("remove");
+		doveadm_print(ctx->socket_path);
+		doveadm_print(line);
+		doveadm_print("");
 	}
 	if (line == NULL) {
 		i_error("Director disconnected unexpectedly");
@@ -732,18 +739,20 @@ static void director_read_ok_reply(struct director_context *ctx)
 	}
 }
 
-static void cmd_director_ring_add(int argc, char *argv[])
+static void cmd_director_ring_add(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
 {
 	struct director_context *ctx;
 	struct ip_addr ip;
 	in_port_t port = 0;
 	string_t *str = t_str_new(64);
 
-	ctx = cmd_director_init(argc, argv, "a:", cmd_director_ring_add);
-	if (argv[optind] == NULL ||
-	    net_addr2ip(argv[optind], &ip) < 0 ||
-	    (argv[optind+1] != NULL && net_str2port(argv[optind+1], &port) < 0))
-		director_cmd_help(cmd_director_ring_add);
+	ctx = cmd_director_init(argc, argv);
+	if (ctx->ip == NULL ||
+	    net_addr2ip(ctx->ip, &ip) < 0 ||
+	    (ctx->port && net_str2port(ctx->port, &port) < 0)) {
+		director_cmd_help(cmd);
+		return;
+	}
 
 	str_printfa(str, "DIRECTOR-ADD\t%s", net_ip2addr(&ip));
 	if (port != 0)
@@ -754,18 +763,20 @@ static void cmd_director_ring_add(int argc, char *argv[])
 	director_disconnect(ctx);
 }
 
-static void cmd_director_ring_remove(int argc, char *argv[])
+static void cmd_director_ring_remove(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
 {
 	struct director_context *ctx;
 	struct ip_addr ip;
 	string_t *str = t_str_new(64);
 	in_port_t port = 0;
 
-	ctx = cmd_director_init(argc, argv, "a:", cmd_director_ring_remove);
-	if (argv[optind] == NULL ||
-	    net_addr2ip(argv[optind], &ip) < 0 ||
-	    (argv[optind+1] != NULL && net_str2port(argv[optind+1], &port) < 0))
-		director_cmd_help(cmd_director_ring_remove);
+	ctx = cmd_director_init(argc, argv);
+	if (ctx->ip == NULL ||
+	    net_addr2ip(ctx->ip, &ip) < 0 ||
+	    (ctx->port != NULL && net_str2port(ctx->port, &port) < 0)) {
+		director_cmd_help(cmd);
+		return;
+	}
 
 	str_printfa(str, "DIRECTOR-REMOVE\t%s", net_ip2addr(&ip));
 	if (port != 0)
@@ -776,13 +787,13 @@ static void cmd_director_ring_remove(int argc, char *argv[])
 	director_disconnect(ctx);
 }
 
-static void cmd_director_ring_status(int argc, char *argv[])
+static void cmd_director_ring_status(const struct doveadm_cmd_ver2 *cmd ATTR_UNUSED, int argc, const struct doveadm_cmd_param *argv)
 {
 	struct director_context *ctx;
 	const char *line, *const *args;
 	unsigned long l;
 
-	ctx = cmd_director_init(argc, argv, "a:", cmd_director_ring_status);
+	ctx = cmd_director_init(argc, argv);
 
 	doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
 	doveadm_print_header_simple("director ip");
@@ -817,46 +828,152 @@ static void cmd_director_ring_status(int argc, char *argv[])
 	director_disconnect(ctx);
 }
 
-struct doveadm_cmd doveadm_cmd_director[] = {
-	{ cmd_director_status, "director status",
-	  "[-a <director socket path>] [<user>]" },
-	{ cmd_director_map, "director map",
-	  "[-a <director socket path>] [-f <users file>] [-h | -u] [<host>]" },
-	{ cmd_director_add, "director add",
-	  "[-a <director socket path>] [-t <tag>] <host> [<vhost count>]" },
-	{ cmd_director_update, "director update",
-	  "[-a <director socket path>] <host> <vhost count>" },
-	{ cmd_director_up, "director up",
-	  "[-a <director socket path>] <host>" },
-	{ cmd_director_down, "director down",
-	  "[-a <director socket path>] <host>" },
-	{ cmd_director_remove, "director remove",
-	  "[-a <director socket path>] <host>" },
-	{ cmd_director_move, "director move",
-	  "[-a <director socket path>] <user> <host>" },
-	{ cmd_director_kick, "director kick",
-	  "[-a <director socket path>] <user>" },
-	{ cmd_director_flush, "director flush",
-	  "[-a <director socket path>] [-f] <host>|all" },
-	{ cmd_director_dump, "director dump",
-	  "[-a <director socket path>]" },
-	{ cmd_director_ring_add, "director ring add",
-	  "[-a <director socket path>] <ip> [<port>]" },
-	{ cmd_director_ring_remove, "director ring remove",
-	  "[-a <director socket path>] <ip> [<port>]" },
-	{ cmd_director_ring_status, "director ring status",
-	  "[-a <director socket path>]" }
+struct doveadm_cmd_ver2 doveadm_cmd_director[] = {
+{
+	.name = "director status",
+	.cmd = cmd_director_status,
+	.usage = "[-a <director socket path>] [<user>] [<tag>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "tag", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "director map",
+	.cmd = cmd_director_map,
+	.usage = "[-a <director socket path>] [-f <users file>] [-h | -u] [<host>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('f', "users-file", CMD_PARAM_ISTREAM, 0)
+DOVEADM_CMD_PARAM('h', "hash-map", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('u', "user-map", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "director add",
+	.cmd = cmd_director_add,
+	.usage = "[-a <director socket path>] [-t <tag>] <host> [<vhost count>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('t', "tag", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "vhost-count", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "director update",
+	.cmd = cmd_director_update,
+	.usage = "[-a <director socket path>] <host> <vhost count>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "vhost-count", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "director up",
+	.cmd = cmd_director_up,
+	.usage = "[-a <director socket path>] <host>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "director down",
+	.cmd = cmd_director_down,
+	.usage = "[-a <director socket path>] <host>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "director remove",
+	.cmd = cmd_director_remove,
+	.usage = "[-a <director socket path>] <host>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "director move",
+	.cmd = cmd_director_move,
+	.usage = "[-a <director socket path>] <user> <host>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "director kick",
+	.cmd = cmd_director_kick,
+	.usage = "[-a <director socket path>] <user>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "director flush",
+	.cmd = cmd_director_flush,
+	.usage = "[-a <director socket path>] [-F] <host>|all",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('F', "force-flush", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "director dump",
+	.cmd = cmd_director_dump,
+	.usage = "[-a <director socket path>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "director ring add",
+	.cmd = cmd_director_ring_add,
+	.usage = "[-a <director socket path>] <ip> [<port>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "ip", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "port", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "director ring remove",
+	.cmd = cmd_director_ring_remove,
+	.usage = "[-a <director socket path>] <ip> [<port>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "ip", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "port", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "director ring status",
+	.cmd = cmd_director_ring_status,
+	.usage = "[-a <director socket path>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAMS_END
+}
 };
 
-static void director_cmd_help(doveadm_command_t *cmd)
+static void director_cmd_help(const struct doveadm_cmd_ver2 *cmd)
 {
 	unsigned int i;
 
 	for (i = 0; i < N_ELEMENTS(doveadm_cmd_director); i++) {
-		if (doveadm_cmd_director[i].cmd == cmd)
-			help(&doveadm_cmd_director[i]);
+		if (doveadm_cmd_director+i == cmd)
+			help_ver2(&doveadm_cmd_director[i]);
 	}
-	i_unreached();
 }
 
 void doveadm_register_director_commands(void)
@@ -864,5 +981,5 @@ void doveadm_register_director_commands(void)
 	unsigned int i;
 
 	for (i = 0; i < N_ELEMENTS(doveadm_cmd_director); i++)
-		doveadm_register_cmd(&doveadm_cmd_director[i]);
+		doveadm_cmd_register_ver2(&doveadm_cmd_director[i]);
 }

From d7dfbe245b042dd1b387908f6cf049a7ba410b74 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 27 May 2016 11:29:09 +0300
Subject: [PATCH 185/450] doveadm mailbox delete: Added -e parameter to delete
 only empty mailboxes.

---
 src/doveadm/doveadm-mail-mailbox.c | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/doveadm/doveadm-mail-mailbox.c b/src/doveadm/doveadm-mail-mailbox.c
index c1f7c2ee060..19ef531480d 100644
--- a/src/doveadm/doveadm-mail-mailbox.c
+++ b/src/doveadm/doveadm-mail-mailbox.c
@@ -34,6 +34,7 @@ struct delete_cmd_context {
 	struct doveadm_mailbox_cmd_context ctx;
 	ARRAY_TYPE(const_string) mailboxes;
 	bool recursive;
+	bool require_empty;
 };
 
 struct rename_cmd_context {
@@ -334,7 +335,7 @@ cmd_mailbox_delete_run(struct doveadm_mail_cmd_context *_ctx,
 	const char *const *namep;
 	ARRAY_TYPE(const_string) recursive_mailboxes;
 	const ARRAY_TYPE(const_string) *mailboxes = &ctx->mailboxes;
-	int ret = 0;
+	int ret = 0, ret2;
 
 	if (ctx->recursive) {
 		t_array_init(&recursive_mailboxes, 32);
@@ -357,7 +358,9 @@ cmd_mailbox_delete_run(struct doveadm_mail_cmd_context *_ctx,
 		ns = mail_namespace_find(user->namespaces, name);
 		box = mailbox_alloc(ns->list, name, 0);
 		storage = mailbox_get_storage(box);
-		if (mailbox_delete(box) < 0) {
+		ret2 = ctx->require_empty ? mailbox_delete_empty(box) :
+			mailbox_delete(box);
+		if (ret2 < 0) {
 			i_error("Can't delete mailbox %s: %s", name,
 				mailbox_get_last_error(box, NULL));
 			doveadm_mail_failed_mailbox(_ctx, box);
@@ -406,6 +409,9 @@ cmd_mailbox_delete_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
 	case 's':
 		ctx->ctx.subscriptions = TRUE;
 		break;
+	case 'e':
+		ctx->require_empty = TRUE;
+		break;
 	default:
 		return FALSE;
 	}
@@ -420,7 +426,7 @@ static struct doveadm_mail_cmd_context *cmd_mailbox_delete_alloc(void)
 	ctx->ctx.ctx.v.init = cmd_mailbox_delete_init;
 	ctx->ctx.ctx.v.run = cmd_mailbox_delete_run;
 	ctx->ctx.ctx.v.parse_arg = cmd_mailbox_delete_parse_arg;
-	ctx->ctx.ctx.getopt_args = "rs";
+	ctx->ctx.ctx.getopt_args = "ers";
 	p_array_init(&ctx->mailboxes, ctx->ctx.ctx.pool, 16);
 	return &ctx->ctx.ctx;
 }
@@ -680,10 +686,11 @@ DOVEADM_CMD_PARAMS_END
 
 struct doveadm_cmd_ver2 doveadm_cmd_mailbox_delete_ver2 = {
 	.name = "mailbox delete",
-        .mail_cmd = cmd_mailbox_delete_alloc,
-        .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-s] <mailbox> [...]",
+	.mail_cmd = cmd_mailbox_delete_alloc,
+	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-e] [-r] [-s] <mailbox> [...]",
 DOVEADM_CMD_PARAMS_START
 DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('e', "require-empty", CMD_PARAM_BOOL, 0)
 DOVEADM_CMD_PARAM('s', "subscriptions", CMD_PARAM_BOOL, 0)
 DOVEADM_CMD_PARAM('r', "recursive", CMD_PARAM_BOOL, 0)
 DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)

From 497b8054e1f4a6ca571dfc1a3e3c11ffc1acadbc Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Mon, 30 May 2016 14:45:09 +0300
Subject: [PATCH 186/450] doveadm-director: Fix commands so that they compile

---
 src/doveadm/doveadm-director.c | 112 ++++++++++++++++-----------------
 1 file changed, 56 insertions(+), 56 deletions(-)

diff --git a/src/doveadm/doveadm-director.c b/src/doveadm/doveadm-director.c
index 516cafabecb..2d6de55d832 100644
--- a/src/doveadm/doveadm-director.c
+++ b/src/doveadm/doveadm-director.c
@@ -40,7 +40,7 @@ struct user_list {
 
 HASH_TABLE_DEFINE_TYPE(user_list, void *, struct user_list *);
 
-static void director_cmd_help(const struct doveadm_cmd_ver2 *cmd ATTR_UNUSED);
+static void director_cmd_help(const struct doveadm_cmd_ver2 *);
 static int director_get_host(const char *host, struct ip_addr **ips_r,
                               unsigned int *ips_count_r) ATTR_WARN_UNUSED_RESULT;
 static void
@@ -92,34 +92,34 @@ static void director_disconnect(struct director_context *ctx)
 }
 
 static struct director_context *
-cmd_director_init(int argc, const struct doveadm_cmd_param *argv)
+cmd_director_init(struct doveadm_cmd_context *cctx)
 {
 	struct director_context *ctx;
 	ctx = t_new(struct director_context, 1);
-	if (!doveadm_cmd_param_str(argc, argv, "socket-path", &(ctx->socket_path)))
+	if (!doveadm_cmd_param_str(cctx, "socket-path", &(ctx->socket_path)))
 		ctx->socket_path = t_strconcat(doveadm_settings->base_dir,
 					"/director-admin", NULL);
 	else
 		ctx->explicit_socket_path = TRUE;
-	if (!doveadm_cmd_param_bool(argc, argv, "user-map", &(ctx->user_map)))
+	if (!doveadm_cmd_param_bool(cctx, "user-map", &(ctx->user_map)))
 		ctx->user_map = FALSE;
-	if (!doveadm_cmd_param_bool(argc, argv, "hash-map", &(ctx->hash_map)))
+	if (!doveadm_cmd_param_bool(cctx, "hash-map", &(ctx->hash_map)))
 		ctx->hash_map = FALSE;
-	if (!doveadm_cmd_param_bool(argc, argv, "force-flush", &(ctx->force_flush)))
+	if (!doveadm_cmd_param_bool(cctx, "force-flush", &(ctx->force_flush)))
 		ctx->force_flush = FALSE;
-	if (!doveadm_cmd_param_istream(argc, argv, "users-file", &(ctx->users_input)))
+	if (!doveadm_cmd_param_istream(cctx, "users-file", &(ctx->users_input)))
 		ctx->users_input = NULL;
-	if (!doveadm_cmd_param_str(argc, argv, "tag", &(ctx->tag)))
+	if (!doveadm_cmd_param_str(cctx, "tag", &(ctx->tag)))
 		ctx->tag = NULL;
-	if (!doveadm_cmd_param_str(argc, argv, "user", &(ctx->user)))
+	if (!doveadm_cmd_param_str(cctx, "user", &(ctx->user)))
 		ctx->user = NULL;
-	if (!doveadm_cmd_param_str(argc, argv, "host", &(ctx->host)))
+	if (!doveadm_cmd_param_str(cctx, "host", &(ctx->host)))
 		ctx->host = NULL;
-	if (!doveadm_cmd_param_str(argc, argv, "ip", &(ctx->ip)))
+	if (!doveadm_cmd_param_str(cctx, "ip", &(ctx->ip)))
 		ctx->ip = NULL;
-	if (!doveadm_cmd_param_str(argc, argv, "port", &(ctx->port)))
+	if (!doveadm_cmd_param_str(cctx, "port", &(ctx->port)))
 		ctx->port = NULL;
-	if (!doveadm_cmd_param_str(argc, argv, "vhost-count", &(ctx->vhost_count)))
+	if (!doveadm_cmd_param_str(cctx, "vhost-count", &(ctx->vhost_count)))
 		ctx->vhost_count = NULL;
 	if (!ctx->user_map)
 		director_connect(ctx);
@@ -174,12 +174,12 @@ cmd_director_status_user(struct director_context *ctx)
 	director_disconnect(ctx);
 }
 
-static void cmd_director_status(const struct doveadm_cmd_ver2 *cmd ATTR_UNUSED, int argc, const struct doveadm_cmd_param *argv)
+static void cmd_director_status(struct doveadm_cmd_context *cctx)
 {
 	struct director_context *ctx;
 	const char *line, *const *args;
 
-	ctx = cmd_director_init(argc, argv);
+	ctx = cmd_director_init(cctx);
 	if (ctx->user != NULL) {
 		cmd_director_status_user(ctx);
 		return;
@@ -311,7 +311,7 @@ static bool ip_find(const struct ip_addr *ips, unsigned int ips_count,
 	return FALSE;
 }
 
-static void cmd_director_map(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
+static void cmd_director_map(struct doveadm_cmd_context *cctx)
 {
 	struct director_context *ctx;
 	const char *line, *const *args;
@@ -321,10 +321,10 @@ static void cmd_director_map(const struct doveadm_cmd_ver2 *cmd, int argc, const
 	struct user_list *user;
 	unsigned int ips_count, user_hash, expires;
 
-	ctx = cmd_director_init(argc, argv);
+	ctx = cmd_director_init(cctx);
 
 	if ((ctx->hash_map && ctx->user_map) && ctx->host == NULL) {
-		director_cmd_help(cmd);
+		director_cmd_help(cctx->cmd);
 	return;
 	}
 
@@ -416,7 +416,7 @@ static void cmd_director_map(const struct doveadm_cmd_ver2 *cmd, int argc, const
 }
 
 static void
-cmd_director_add_or_update(const struct doveadm_cmd_ver2 *dcmd, int argc, const struct doveadm_cmd_param *argv, bool update)
+cmd_director_add_or_update(struct doveadm_cmd_context *cctx, bool update)
 {
 	const char *director_cmd = update ? "HOST-UPDATE" : "HOST-SET";
 	struct director_context *ctx;
@@ -425,20 +425,20 @@ cmd_director_add_or_update(const struct doveadm_cmd_ver2 *dcmd, int argc, const
 	const char *line, *host;
 	string_t *cmd;
 
-	ctx = cmd_director_init(argc, argv);
+	ctx = cmd_director_init(cctx);
 	if (ctx->tag != NULL && ctx->tag[0] == '\0')
 		ctx->tag = NULL;
 	if (ctx->host == NULL) {
-		director_cmd_help(dcmd);
+		director_cmd_help(cctx->cmd);
 		return;
 	}
 	if (ctx->vhost_count != NULL) {
 		if (str_to_uint(ctx->vhost_count, &vhost_count) < 0) {
-			director_cmd_help(dcmd);
+			director_cmd_help(cctx->cmd);
 			return;
 		}
 	} else if (update) {
-		director_cmd_help(dcmd);
+		director_cmd_help(cctx->cmd);
 		return;
 	}
 
@@ -475,29 +475,29 @@ cmd_director_add_or_update(const struct doveadm_cmd_ver2 *dcmd, int argc, const
 	director_disconnect(ctx);
 }
 
-static void cmd_director_add(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
+static void cmd_director_add(struct doveadm_cmd_context *cctx)
 {
-	cmd_director_add_or_update(cmd, argc, argv, FALSE);
+	cmd_director_add_or_update(cctx, FALSE);
 }
 
-static void cmd_director_update(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
+static void cmd_director_update(struct doveadm_cmd_context *cctx)
 {
-	cmd_director_add_or_update(cmd, argc, argv, TRUE);
+	cmd_director_add_or_update(cctx, TRUE);
 }
 
 static void
-cmd_director_ipcmd(const char *cmd_name, const struct doveadm_cmd_ver2 *cmd, const char *success_result,
-	int argc, const struct doveadm_cmd_param *argv)
+cmd_director_ipcmd(const char *cmd_name, const char *success_result,
+	struct doveadm_cmd_context *cctx)
 {
 	struct director_context *ctx;
 	struct ip_addr *ips;
 	unsigned int i, ips_count;
 	const char *host, *line;
 
-	ctx = cmd_director_init(argc, argv);
+	ctx = cmd_director_init(cctx);
 	host = ctx->host;
 	if (host == NULL) {
-		director_cmd_help(cmd);
+		director_cmd_help(cctx->cmd);
 		return;
 	}
 
@@ -524,31 +524,31 @@ cmd_director_ipcmd(const char *cmd_name, const struct doveadm_cmd_ver2 *cmd, con
 	director_disconnect(ctx);
 }
 
-static void cmd_director_remove(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
+static void cmd_director_remove(struct doveadm_cmd_context *cctx)
 {
-	cmd_director_ipcmd("HOST-REMOVE", cmd, "removed", argc, argv);
+	cmd_director_ipcmd("HOST-REMOVE", "removed", cctx);
 }
 
-static void cmd_director_up(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
+static void cmd_director_up(struct doveadm_cmd_context *cctx)
 {
-	cmd_director_ipcmd("HOST-UP", cmd, "up", argc, argv);
+	cmd_director_ipcmd("HOST-UP", "up", cctx);
 }
 
-static void cmd_director_down(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
+static void cmd_director_down(struct doveadm_cmd_context *cctx)
 {
-	cmd_director_ipcmd("HOST-DOWN", cmd, "down", argc, argv);
+	cmd_director_ipcmd("HOST-DOWN", "down", cctx);
 }
 
-static void cmd_director_move(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
+static void cmd_director_move(struct doveadm_cmd_context *cctx)
 {
 	struct director_context *ctx;
 	struct ip_addr *ips;
 	unsigned int ips_count, user_hash;
 	const char *line, *ip_str;
 
-	ctx = cmd_director_init(argc, argv);
+	ctx = cmd_director_init(cctx);
 	if (ctx->user == NULL || ctx->host == NULL) {
-		director_cmd_help(cmd);
+		director_cmd_help(cctx->cmd);
 		return;
 	}
 
@@ -579,14 +579,14 @@ static void cmd_director_move(const struct doveadm_cmd_ver2 *cmd, int argc, cons
 	director_disconnect(ctx);
 }
 
-static void cmd_director_kick(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
+static void cmd_director_kick(struct doveadm_cmd_context *cctx)
 {
 	struct director_context *ctx;
 	const char *line;
 
-	ctx = cmd_director_init(argc, argv);
+	ctx = cmd_director_init(cctx);
 	if (ctx->user == NULL) {
-		director_cmd_help(cmd);
+		director_cmd_help(cctx->cmd);
 		return;
 	}
 
@@ -624,7 +624,7 @@ static void cmd_director_flush_all(struct director_context *ctx)
 	director_disconnect(ctx);
 }
 
-static void cmd_director_flush(const struct doveadm_cmd_ver2 *cmd ATTR_UNUSED, int argc, const struct doveadm_cmd_param *argv)
+static void cmd_director_flush(struct doveadm_cmd_context *cctx)
 {
 	struct director_context *ctx;
 	struct ip_addr *ips;
@@ -632,9 +632,9 @@ static void cmd_director_flush(const struct doveadm_cmd_ver2 *cmd ATTR_UNUSED, i
 	struct ip_addr ip;
 	const char *line;
 
-	ctx = cmd_director_init(argc, argv);
+	ctx = cmd_director_init(cctx);
 	if (ctx->host == NULL) {
-		director_cmd_help(cmd);
+		director_cmd_help(cctx->cmd);
 		return;
 	}
 
@@ -672,12 +672,12 @@ static void cmd_director_flush(const struct doveadm_cmd_ver2 *cmd ATTR_UNUSED, i
 	director_disconnect(ctx);
 }
 
-static void cmd_director_dump(const struct doveadm_cmd_ver2 *cmd ATTR_UNUSED, int argc, const struct doveadm_cmd_param *argv)
+static void cmd_director_dump(struct doveadm_cmd_context *cctx)
 {
 	struct director_context *ctx;
 	const char *line, *const *args;
 
-	ctx = cmd_director_init(argc, argv);
+	ctx = cmd_director_init(cctx);
 
 	doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED);
 	if (ctx->explicit_socket_path)
@@ -739,18 +739,18 @@ static void director_read_ok_reply(struct director_context *ctx)
 	}
 }
 
-static void cmd_director_ring_add(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
+static void cmd_director_ring_add(struct doveadm_cmd_context *cctx)
 {
 	struct director_context *ctx;
 	struct ip_addr ip;
 	in_port_t port = 0;
 	string_t *str = t_str_new(64);
 
-	ctx = cmd_director_init(argc, argv);
+	ctx = cmd_director_init(cctx);
 	if (ctx->ip == NULL ||
 	    net_addr2ip(ctx->ip, &ip) < 0 ||
 	    (ctx->port && net_str2port(ctx->port, &port) < 0)) {
-		director_cmd_help(cmd);
+		director_cmd_help(cctx->cmd);
 		return;
 	}
 
@@ -763,18 +763,18 @@ static void cmd_director_ring_add(const struct doveadm_cmd_ver2 *cmd, int argc,
 	director_disconnect(ctx);
 }
 
-static void cmd_director_ring_remove(const struct doveadm_cmd_ver2 *cmd, int argc, const struct doveadm_cmd_param *argv)
+static void cmd_director_ring_remove(struct doveadm_cmd_context *cctx)
 {
 	struct director_context *ctx;
 	struct ip_addr ip;
 	string_t *str = t_str_new(64);
 	in_port_t port = 0;
 
-	ctx = cmd_director_init(argc, argv);
+	ctx = cmd_director_init(cctx);
 	if (ctx->ip == NULL ||
 	    net_addr2ip(ctx->ip, &ip) < 0 ||
 	    (ctx->port != NULL && net_str2port(ctx->port, &port) < 0)) {
-		director_cmd_help(cmd);
+		director_cmd_help(cctx->cmd);
 		return;
 	}
 
@@ -787,13 +787,13 @@ static void cmd_director_ring_remove(const struct doveadm_cmd_ver2 *cmd, int arg
 	director_disconnect(ctx);
 }
 
-static void cmd_director_ring_status(const struct doveadm_cmd_ver2 *cmd ATTR_UNUSED, int argc, const struct doveadm_cmd_param *argv)
+static void cmd_director_ring_status(struct doveadm_cmd_context *cctx)
 {
 	struct director_context *ctx;
 	const char *line, *const *args;
 	unsigned long l;
 
-	ctx = cmd_director_init(argc, argv);
+	ctx = cmd_director_init(cctx);
 
 	doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
 	doveadm_print_header_simple("director ip");

From 141f09fb6660901c8eac78077713ce3e2f28b8f2 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Wed, 27 Apr 2016 14:08:00 +0300
Subject: [PATCH 187/450] lib-dcrypt: Initial implementation

---
 configure.ac                                  |    1 +
 src/Makefile.am                               |    1 +
 src/lib-dcrypt/Makefile.am                    |   64 +
 src/lib-dcrypt/dcrypt-gnutls.c                |  493 ++++
 src/lib-dcrypt/dcrypt-iostream-private.h      |    8 +
 src/lib-dcrypt/dcrypt-openssl.c               | 2015 +++++++++++++++++
 src/lib-dcrypt/dcrypt-private.h               |  104 +
 src/lib-dcrypt/dcrypt.c                       |  275 +++
 src/lib-dcrypt/dcrypt.h                       |  204 ++
 src/lib-dcrypt/istream-decrypt.c              |  873 +++++++
 src/lib-dcrypt/istream-decrypt.h              |   30 +
 src/lib-dcrypt/ostream-encrypt.c              |  686 ++++++
 src/lib-dcrypt/ostream-encrypt.h              |   38 +
 src/lib-dcrypt/sample-v1.bin                  |  Bin 0 -> 2281 bytes
 src/lib-dcrypt/sample-v2.bin                  |  Bin 0 -> 13757 bytes
 src/lib-dcrypt/test-crypto.c                  |  305 +++
 src/lib-dcrypt/test-stream.c                  |  232 ++
 .../index/dbox-single/sdbox-file.c            |    3 +
 18 files changed, 5332 insertions(+)
 create mode 100644 src/lib-dcrypt/Makefile.am
 create mode 100644 src/lib-dcrypt/dcrypt-gnutls.c
 create mode 100644 src/lib-dcrypt/dcrypt-iostream-private.h
 create mode 100644 src/lib-dcrypt/dcrypt-openssl.c
 create mode 100644 src/lib-dcrypt/dcrypt-private.h
 create mode 100644 src/lib-dcrypt/dcrypt.c
 create mode 100644 src/lib-dcrypt/dcrypt.h
 create mode 100644 src/lib-dcrypt/istream-decrypt.c
 create mode 100644 src/lib-dcrypt/istream-decrypt.h
 create mode 100644 src/lib-dcrypt/ostream-encrypt.c
 create mode 100644 src/lib-dcrypt/ostream-encrypt.h
 create mode 100644 src/lib-dcrypt/sample-v1.bin
 create mode 100755 src/lib-dcrypt/sample-v2.bin
 create mode 100644 src/lib-dcrypt/test-crypto.c
 create mode 100644 src/lib-dcrypt/test-stream.c

diff --git a/configure.ac b/configure.ac
index dff5df48d5c..0bcdceba293 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2860,6 +2860,7 @@ src/lib-sql/Makefile
 src/lib-auth/Makefile
 src/lib-charset/Makefile
 src/lib-compression/Makefile
+src/lib-dcrypt/Makefile
 src/lib-dict/Makefile
 src/lib-dict-extra/Makefile
 src/lib-dns/Makefile
diff --git a/src/Makefile.am b/src/Makefile.am
index 39c62d213a7..b456be5db5a 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -9,6 +9,7 @@ LIBDOVECOT_SUBDIRS = \
 	lib-auth \
 	lib-master \
 	lib-charset \
+	lib-dcrypt \
 	lib-dns \
 	lib-dict \
 	lib-sasl \
diff --git a/src/lib-dcrypt/Makefile.am b/src/lib-dcrypt/Makefile.am
new file mode 100644
index 00000000000..73c5c1d9a98
--- /dev/null
+++ b/src/lib-dcrypt/Makefile.am
@@ -0,0 +1,64 @@
+noinst_LTLIBRARIES = libdcrypt.la
+pkglib_LTLIBRARIES =
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-test
+
+libdcrypt_la_SOURCES = \
+	dcrypt.c \
+	istream-decrypt.c \
+	ostream-encrypt.c
+
+libdcrypt_la_CFLAGS = $(AM_CPPFLAGS) \
+	-DDCRYPT_MODULE_DIR=\"$(pkglibdir)\"
+
+if BUILD_OPENSSL
+pkglib_LTLIBRARIES += libdcrypt_openssl.la
+libdcrypt_openssl_la_SOURCES = dcrypt-openssl.c
+libdcrypt_openssl_la_DEPENDENCIES = $(SSL_LIBS)
+libdcrypt_openssl_la_LDFLAGS = -module -avoid-version -shared $(SSL_LIBS)
+libdcrypt_openssl_la_CFLAGS = $(AM_CPPFLAGS) \
+	$(SSL_CFLAGS)
+endif
+
+headers = \
+	dcrypt.h \
+	dcrypt-iostream-private.h \
+	dcrypt-private.h \
+	ostream-encrypt.h \
+	istream-decrypt.h
+
+pkginc_libdir=$(pkgincludedir)
+pkginc_lib_HEADERS = $(headers)
+
+EXTRA_DIST = \
+	sample-v1.bin \
+	sample-v2.bin
+
+check_PROGRAMS = test-crypto test-stream
+
+check: check-am check-test
+
+check-test: all-am
+	for bin in $(check_PROGRAMS); do \
+	  if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
+	done
+
+
+noinst_DATA = \
+	sample-v1.bin \
+	sample-v2.bin
+
+LIBDOVECOT_TEST = \
+	../lib-test/libtest.la \
+	../lib/liblib.la \
+	$(MODULE_LIBS)
+
+test_crypto_LDADD = $(LIBDOVECOT_TEST)
+test_crypto_CFLAGS = $(AM_CPPFLAGS) -DDCRYPT_MODULE_DIR=\"$(top_srcdir)/src/lib-dcrypt/.libs\"
+test_crypto_SOURCES = $(libdcrypt_la_SOURCES) test-crypto.c
+
+test_stream_LDADD = $(LIBDOVECOT_TEST)
+test_stream_CFLAGS = $(AM_CPPFLAGS) -DDCRYPT_MODULE_DIR=\"$(top_srcdir)/src/lib-dcrypt/.libs\"
+test_stream_SOURCES = $(libdcrypt_la_SOURCES) test-stream.c
diff --git a/src/lib-dcrypt/dcrypt-gnutls.c b/src/lib-dcrypt/dcrypt-gnutls.c
new file mode 100644
index 00000000000..7185435aa45
--- /dev/null
+++ b/src/lib-dcrypt/dcrypt-gnutls.c
@@ -0,0 +1,493 @@
+#include "lib.h"
+#include "buffer.h"
+#include "randgen.h"
+#include "array.h"
+#include "hash-method.h"
+#include "pkcs5.h"
+#include "module-dir.h"
+#include <gnutls/gnutls.h>
+#include <gnutls/compat.h>
+#include <gnutls/abstract.h>
+#include <gnutls/crypto.h>
+#include <nettle/version.h>
+#include <nettle/hmac.h>
+#include <nettle/pbkdf2.h>
+#include <nettle/ecc.h>
+
+#include "dcrypt.h"
+#include "dcrypt-private.h"
+
+struct dcrypt_context_symmetric {
+	pool_t pool;
+	gnutls_cipher_hd_t ctx;
+	gnutls_cipher_algorithm_t cipher;
+	gnutls_datum_t key;
+	gnutls_datum_t iv;
+	enum dcrypt_sym_mode mode;
+};
+
+struct dcrypt_context_hmac {
+	pool_t pool;
+	gnutls_hmac_hd_t ctx;
+	gnutls_mac_algorithm_t md;
+	gnutls_datum_t key;
+	size_t klen;
+};
+
+struct dcrypt_public_key {
+	void *ctx;
+};
+
+struct dcrypt_private_key {
+	void *ctx;
+};
+
+static
+int dcrypt_gnutls_private_to_public_key(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r, const char **error_r);
+
+static
+int dcrypt_gnutls_error(int ec, const char **error_r)
+{
+	i_assert(ec < 0);
+	if(error_r != NULL) {
+		*error_r = gnutls_strerror(ec);
+	}
+	return -1;
+}
+
+static
+int dcrypt_gnutls_ctx_sym_create(const char *algorithm, enum dcrypt_sym_mode mode, struct dcrypt_context_symmetric **ctx_r, const char **error_r)
+{
+	gnutls_cipher_algorithm_t cipher = gnutls_cipher_get_id(algorithm);
+	if(cipher == GNUTLS_CIPHER_UNKNOWN) return dcrypt_gnutls_error(cipher, error_r);
+	pool_t pool = pool_alloconly_create("dcrypt gnutls", 128);
+	struct dcrypt_context_symmetric *ctx = p_new(pool, struct dcrypt_context_symmetric, 1);
+	ctx->pool = pool;
+	ctx->cipher = cipher;
+	ctx->mode = mode;
+	*ctx_r = ctx;
+	return 0;
+}
+
+static
+int dcrypt_gnutls_ctx_sym_destroy(struct dcrypt_context_symmetric **ctx)
+{
+	pool_t pool =(*ctx)->pool;
+	gnutls_cipher_deinit((*ctx)->ctx);
+	pool_unref(&pool);
+	*ctx = NULL;
+	return 0;
+}
+
+static
+void dcrypt_gnutls_ctx_sym_set_key(struct dcrypt_context_symmetric *ctx, const unsigned char *key, size_t key_len)
+{
+	if(ctx->key.data != NULL) p_free(ctx->pool, ctx->key.data);
+	ctx->key.size = I_MIN(key_len,(size_t)gnutls_cipher_get_key_size(ctx->cipher));
+	ctx->key.data = p_malloc(ctx->pool, ctx->key.size);
+	memcpy(ctx->key.data, key, ctx->key.size);
+}
+
+static
+void dcrypt_gnutls_ctx_sym_set_iv(struct dcrypt_context_symmetric *ctx, const unsigned char *iv, size_t iv_len)
+{
+	if(ctx->iv.data != NULL) p_free(ctx->pool, ctx->iv.data);
+	ctx->iv.size = I_MIN(iv_len,(size_t)gnutls_cipher_get_iv_size(ctx->cipher));
+	ctx->iv.data = p_malloc(ctx->pool, ctx->iv.size);
+	memcpy(ctx->iv.data, iv, ctx->iv.size);
+}
+
+static
+void dcrypt_gnutls_ctx_sym_set_key_iv_random(struct dcrypt_context_symmetric *ctx)
+{
+	if(ctx->key.data != NULL) p_free(ctx->pool, ctx->key.data);
+	if(ctx->iv.data != NULL) p_free(ctx->pool, ctx->iv.data);
+	ctx->key.data = p_malloc(ctx->pool, gnutls_cipher_get_key_size(ctx->cipher));
+	random_fill(ctx->key.data, gnutls_cipher_get_key_size(ctx->cipher));
+	ctx->key.size = gnutls_cipher_get_key_size(ctx->cipher);
+	ctx->iv.data = p_malloc(ctx->pool, gnutls_cipher_get_iv_size(ctx->cipher));
+	random_fill(ctx->iv.data, gnutls_cipher_get_iv_size(ctx->cipher));
+	ctx->iv.size = gnutls_cipher_get_iv_size(ctx->cipher);
+}
+
+static
+int dcrypt_gnutls_ctx_sym_get_key(struct dcrypt_context_symmetric *ctx, buffer_t *key)
+{
+	if(ctx->key.data == NULL) return -1;
+	buffer_append(key, ctx->key.data, ctx->key.size);
+	return 0;
+}
+static
+int dcrypt_gnutls_ctx_sym_get_iv(struct dcrypt_context_symmetric *ctx, buffer_t *iv)
+{
+	if(ctx->iv.data == NULL) return -1;
+	buffer_append(iv, ctx->iv.data, ctx->iv.size);
+	return 0;
+}
+
+static
+int dcrypt_gnutls_ctx_sym_get_key_length(struct dcrypt_context_symmetric *ctx)
+{
+	return gnutls_cipher_get_iv_size(ctx->cipher);
+}
+static
+int dcrypt_gnutls_ctx_sym_get_iv_length(struct dcrypt_context_symmetric *ctx)
+{
+	return gnutls_cipher_get_iv_size(ctx->cipher);
+}
+static
+int dcrypt_gnutls_ctx_sym_get_block_size(struct dcrypt_context_symmetric *ctx)
+{
+	return gnutls_cipher_get_block_size(ctx->cipher);
+}
+
+static
+int dcrypt_gnutls_ctx_sym_init(struct dcrypt_context_symmetric *ctx, const char **error_r)
+{
+	int ec;
+	ec = gnutls_cipher_init(&(ctx->ctx), ctx->cipher, &ctx->key, &ctx->iv);
+	if(ec < 0) return dcrypt_gnutls_error(ec, error_r);
+	return 0;
+}
+
+static
+int dcrypt_gnutls_ctx_sym_update(struct dcrypt_context_symmetric *ctx, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r)
+{
+	int ec;
+	size_t outl = gnutls_cipher_get_block_size(ctx->cipher);
+	unsigned char buf[outl];
+	ec = gnutls_cipher_encrypt2(ctx->ctx, data, data_len, buf, outl);
+	if(ec < 0) return dcrypt_gnutls_error(ec, error_r);
+	buffer_append(result, buf, outl);
+	return ec;
+}
+
+static
+int dcrypt_gnutls_ctx_sym_final(struct dcrypt_context_symmetric *ctx, buffer_t *result, const char **error_r)
+{
+	return dcrypt_gnutls_ctx_sym_update(ctx, (const unsigned char*)"", 0, result, error_r);
+}
+
+
+static
+int dcrypt_gnutls_ctx_hmac_create(const char *algorithm, struct dcrypt_context_hmac **ctx_r, const char **error_r)
+{
+	gnutls_mac_algorithm_t md = gnutls_mac_get_id(algorithm);
+	if (md == GNUTLS_MAC_UNKNOWN) return dcrypt_gnutls_error(md, error_r);
+	pool_t pool = pool_alloconly_create("dcrypt gnutls", 128);
+	struct dcrypt_context_hmac *ctx = p_new(pool, struct dcrypt_context_hmac, 1);
+	ctx->pool = pool;
+	ctx->md = md;
+	*ctx_r = ctx;
+	return 0;
+}
+
+static
+int dcrypt_gnutls_ctx_hmac_destroy(struct dcrypt_context_hmac **ctx)
+{
+	pool_t pool = (*ctx)->pool;
+	gnutls_hmac_deinit((*ctx)->ctx, NULL);
+	pool_unref(&pool);
+	*ctx = NULL;
+	return 0;
+}
+
+
+static
+void dcrypt_gnutls_ctx_hmac_set_key(struct dcrypt_context_hmac *ctx, const unsigned char *key, size_t key_len)
+{
+        if(ctx->key.data != NULL) p_free(ctx->pool, ctx->key.data);
+        ctx->key.size = I_MIN(key_len,(size_t)gnutls_hmac_get_len(ctx->md));
+        ctx->key.data = p_malloc(ctx->pool, ctx->key.size);
+        memcpy(ctx->key.data, key, ctx->key.size);
+}
+
+static
+int dcrypt_gnutls_ctx_hmac_get_key(struct dcrypt_context_hmac *ctx, buffer_t *key)
+{
+	if (ctx->key.data == NULL) return -1;
+	buffer_append(key, ctx->key.data, ctx->key.size);
+	return 0;
+}
+
+static
+int dcrypt_gnutls_ctx_hmac_init(struct dcrypt_context_hmac *ctx, const char **error_r)
+{
+	int ec;
+	ec = gnutls_hmac_init(&(ctx->ctx), ctx->md, ctx->key.data, ctx->key.size);
+	if (ec < 0) return dcrypt_gnutls_error(ec, error_r);
+	return 0;
+}
+static
+int dcrypt_gnutls_ctx_hmac_update(struct dcrypt_context_hmac *ctx, const unsigned char *data, size_t data_len, const char **error_r)
+{
+	int ec;
+	if ((ec = gnutls_hmac(ctx->ctx, data, data_len)) != 0)
+		return dcrypt_gnutls_error(ec, error_r);
+	return 0;
+}
+static
+int dcrypt_gnutls_ctx_hmac_final(struct dcrypt_context_hmac *ctx, buffer_t *result, const char **error_r)
+{
+	size_t hlen = gnutls_hmac_get_len(ctx->md);
+	unsigned char buf[hlen];
+	gnutls_hmac_output(ctx->ctx, buf);
+	buffer_append(result, buf, hlen);
+	return 0;
+}
+
+static
+int dcrypt_gnutls_ecdh_derive_secret(struct dcrypt_public_key *peer_key, buffer_t *R, buffer_t *S, const char **error_r)
+{
+
+}
+
+static
+int dcrypt_gnutls_pbkdf2(const unsigned char *password, size_t password_len, const unsigned char *salt, size_t salt_len, const char *algorithm,
+	unsigned int rounds, buffer_t *result, unsigned int result_len, const char **error_r)
+{
+	unsigned char buf[result_len];
+	/* only sha1 or sha256 is supported */
+	if (strncasecmp(algorithm, "sha1", 4) == 0) {
+		pbkdf2_hmac_sha1(password_len, password, rounds, salt_len, salt, result_len, buf);
+	} else if (strncasecmp(algorithm, "sha256", 6) == 0) {
+		pbkdf2_hmac_sha256(password_len, password, rounds, salt_len, salt, result_len, buf);
+	} else if (strncasecmp(algorithm, "sha512", 6) == 0) {
+		struct hmac_sha512_ctx ctx;
+		hmac_sha512_set_key(&ctx, password_len, password);
+		PBKDF2(&ctx, hmac_sha512_update, hmac_sha512_digest, 64, rounds, salt_len, salt, result_len, buf);
+		memset(&ctx, 0, sizeof(ctx));
+	} else {
+		*error_r = "Unsupported algorithm";
+		return -1;
+	}
+	buffer_append(result, buf, result_len);
+	memset(buf, 0, sizeof(buf));
+	return 0;
+}
+
+static
+int dcrypt_gnutls_generate_keypair(struct dcrypt_keypair *pair_r, enum dcrypt_key_type kind, unsigned int bits, const char *curve, const char **error_r)
+{
+	gnutls_pk_algorithm_t pk_algo;
+	gnutls_ecc_curve_t pk_curve;
+
+        if (kind == DCRYPT_KEY_EC) {
+		pk_curve = gnutls_ecc_curve_get_id(curve);
+		if (pk_curve == GNUTLS_ECC_CURVE_INVALID) {
+			*error_r = "Invalid curve";
+			return -1;
+		}
+		bits = GNUTLS_CURVE_TO_BITS(pk_curve);
+#if GNUTLS_VERSION_NUMBER >= 0x030500
+		pk_algo = gnutls_curve_get_pk(pk_curve);
+#else
+		pk_algo = GNUTLS_PK_EC;
+#endif 
+        } else if (kind == DCRYPT_KEY_RSA) {
+                pk_algo = gnutls_pk_get_id("RSA");
+        } else {
+		*error_r = "Unsupported key type";
+		return -1;
+	}
+
+	int ec;
+	gnutls_privkey_t priv;
+	if ((ec = gnutls_privkey_init(&priv)) != GNUTLS_E_SUCCESS) return dcrypt_gnutls_error(ec, error_r);
+#if GNUTLS_VERSION_NUMBER >= 0x030500
+	gnutls_privkey_set_flags(priv, GNUTLS_PRIVKEY_FLAG_EXPORT_COMPAT);
+#endif
+	ec = gnutls_privkey_generate(priv, pk_algo, bits, 0);
+	if (ec != GNUTLS_E_SUCCESS) {
+		gnutls_privkey_deinit(priv);
+		return dcrypt_gnutls_error(ec, error_r);
+	}
+
+	pair_r->priv = (struct dcrypt_private_key*)priv;
+
+	return dcrypt_gnutls_private_to_public_key(pair_r->priv, &(pair_r->pub), error_r);
+} 
+
+static
+int dcrypt_gnutls_load_private_key(struct dcrypt_private_key **key_r, const unsigned char *data, size_t data_len, dcrypt_password_cb *cb, void *ctx, const char **error_r)
+{
+
+}
+static
+int dcrypt_gnutls_load_public_key(struct dcrypt_public_key **key_r, const unsigned char *data, size_t data_len, const char **error_r)
+{
+
+}
+
+static
+int dcrypt_gnutls_store_private_key(struct dcrypt_private_key *key, const char *cipher, buffer_t *destination, dcrypt_password_cb *cb, void *ctx, const char **error_r)
+{
+	gnutls_privkey_t priv = (gnutls_privkey_t)key;
+	gnutls_x509_privkey_t xkey;
+	gnutls_privkey_export_x509(priv, &xkey);
+	/* then export PEM */
+	size_t outl = 0;
+	gnutls_x509_privkey_export_pkcs8(xkey, GNUTLS_X509_FMT_PEM, NULL, 0, NULL, &outl);
+	char buffer[outl];
+	gnutls_x509_privkey_export_pkcs8(xkey, GNUTLS_X509_FMT_PEM, NULL, 0, buffer, &outl);
+	buffer_append(destination, buffer, outl);
+	memset(buffer, 0, sizeof(buffer));
+	return 0;
+}
+
+static
+int dcrypt_gnutls_store_public_key(struct dcrypt_public_key *key, buffer_t *destination, const char **error_r)
+{
+	gnutls_pubkey_t pub = (gnutls_pubkey_t)key;
+	size_t outl = 0;
+	gnutls_pubkey_export(pub, GNUTLS_X509_FMT_PEM, NULL, &outl);
+	char buffer[outl];
+	gnutls_pubkey_export(pub, GNUTLS_X509_FMT_PEM, buffer, &outl);
+	buffer_append(destination, buffer, outl);
+	return 0;
+}
+
+static
+int dcrypt_gnutls_private_to_public_key(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r, const char **error_r)
+{
+	int ec;
+
+	gnutls_privkey_t priv = (gnutls_privkey_t)priv_key;
+	if (gnutls_privkey_get_pk_algorithm(priv, NULL) == GNUTLS_PK_RSA) {
+		gnutls_datum_t m,e;
+		/* do not extract anything we don't need */
+		ec = gnutls_privkey_export_rsa_raw(priv, &m, &e, NULL, NULL, NULL, NULL, NULL, NULL);
+		if (ec != GNUTLS_E_SUCCESS) return dcrypt_gnutls_error(ec, error_r);
+		gnutls_pubkey_t pub;
+		gnutls_pubkey_init(&pub);
+		ec = gnutls_pubkey_import_rsa_raw(pub, &m, &e);
+		gnutls_free(m.data);
+		gnutls_free(e.data);
+		if (ec < 0) {
+			gnutls_pubkey_deinit(pub);
+			return dcrypt_gnutls_error(ec, error_r);
+		}
+		*pub_key_r = (struct dcrypt_public_key*)pub;
+		return 0;
+	} else if (gnutls_privkey_get_pk_algorithm(priv, NULL) == GNUTLS_PK_EC) {
+		gnutls_ecc_curve_t curve;
+		gnutls_datum_t x,y,k;
+		ec = gnutls_privkey_export_ecc_raw(priv, &curve, &x, &y, NULL);
+		if (ec != GNUTLS_E_SUCCESS) return dcrypt_gnutls_error(ec, error_r);
+		gnutls_pubkey_t pub;
+		gnutls_pubkey_init(&pub);
+		ec = gnutls_pubkey_import_ecc_raw(pub, curve, &x, &y);
+		gnutls_free(x.data);
+		gnutls_free(y.data);
+		if (ec < 0) {
+			gnutls_pubkey_deinit(pub);
+			return dcrypt_gnutls_error(ec, error_r);
+		}
+		*pub_key_r = (struct dcrypt_public_key*)pub;
+		return 0;
+	}
+
+	return -1;
+}
+
+static
+void dcrypt_gnutls_free_public_key(struct dcrypt_public_key **key)
+{
+	gnutls_pubkey_deinit((gnutls_pubkey_t)*key);
+	*key = NULL;
+}
+static
+void dcrypt_gnutls_free_private_key(struct dcrypt_private_key **key)
+{
+	gnutls_privkey_deinit((gnutls_privkey_t)*key);
+	*key = NULL;
+}
+static
+void dcrypt_gnutls_free_keypair(struct dcrypt_keypair *keypair)
+{
+	dcrypt_gnutls_free_public_key(&(keypair->pub));
+	dcrypt_gnutls_free_private_key(&(keypair->priv));
+}
+
+static
+int dcrypt_gnutls_rsa_encrypt(struct dcrypt_public_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r)
+{
+
+}
+static
+int dcrypt_gnutls_rsa_decrypt(struct dcrypt_private_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r)
+{
+
+}
+
+static
+int dcrypt_gnutls_oid_keytype(const unsigned char *oid, size_t oid_len, enum dcrypt_key_type *key_type, const char **error_r)
+{
+
+}
+static
+int dcrypt_gnutls_keytype_oid(enum dcrypt_key_type key_type, buffer_t *oid, const char **error_r)
+{
+
+}
+
+static
+const char *dcrypt_gnutls_oid2name(const unsigned char *oid, size_t oid_len, const char **error_r)
+{
+}
+
+static
+int dcrypt_gnutls_name2oid(const char *name, buffer_t *oid, const char **error_r)
+{
+
+}
+
+static struct dcrypt_vfs dcrypt_gnutls_vfs = {
+	.ctx_sym_create = dcrypt_gnutls_ctx_sym_create,
+	.ctx_sym_destroy = dcrypt_gnutls_ctx_sym_destroy,
+	.ctx_sym_set_key = dcrypt_gnutls_ctx_sym_set_key,
+	.ctx_sym_set_iv = dcrypt_gnutls_ctx_sym_set_iv,
+	.ctx_sym_set_key_iv_random = dcrypt_gnutls_ctx_sym_set_key_iv_random,
+	.ctx_sym_get_key = dcrypt_gnutls_ctx_sym_get_key,
+	.ctx_sym_get_iv = dcrypt_gnutls_ctx_sym_get_iv,
+	.ctx_sym_get_key_length = dcrypt_gnutls_ctx_sym_get_key_length,
+	.ctx_sym_get_iv_length = dcrypt_gnutls_ctx_sym_get_iv_length,
+	.ctx_sym_init = dcrypt_gnutls_ctx_sym_init,
+	.ctx_sym_update = dcrypt_gnutls_ctx_sym_update,
+	.ctx_sym_final = dcrypt_gnutls_ctx_sym_final,
+	.ctx_hmac_create = dcrypt_gnutls_ctx_hmac_create,
+	.ctx_hmac_destroy = dcrypt_gnutls_ctx_hmac_destroy,
+	.ctx_hmac_set_key = dcrypt_gnutls_ctx_hmac_set_key,
+	.ctx_hmac_get_key = dcrypt_gnutls_ctx_hmac_get_key,
+	.ctx_hmac_init = dcrypt_gnutls_ctx_hmac_init,
+	.ctx_hmac_update = dcrypt_gnutls_ctx_hmac_update,
+	.ctx_hmac_final = dcrypt_gnutls_ctx_hmac_final,
+//	.ecdh_derive_secret = dcrypt_gnutls_ecdh_derive_secret,
+	.pbkdf2 = dcrypt_gnutls_pbkdf2,
+	.generate_keypair = dcrypt_gnutls_generate_keypair,
+	.load_private_key = dcrypt_gnutls_load_private_key,
+	.load_public_key = dcrypt_gnutls_load_public_key,
+	.store_private_key = dcrypt_gnutls_store_private_key,
+	.store_public_key = dcrypt_gnutls_store_public_key,
+	.private_to_public_key = dcrypt_gnutls_private_to_public_key,
+	.free_keypair = dcrypt_gnutls_free_keypair,
+	.free_public_key = dcrypt_gnutls_free_public_key,
+	.free_private_key = dcrypt_gnutls_free_private_key,
+	.rsa_encrypt = dcrypt_gnutls_rsa_encrypt,
+	.rsa_decrypt = dcrypt_gnutls_rsa_decrypt,
+	.oid_keytype = dcrypt_gnutls_oid_keytype,
+	.keytype_oid = dcrypt_gnutls_keytype_oid,
+	.oid2name = dcrypt_gnutls_oid2name,
+	.name2oid = dcrypt_gnutls_name2oid
+};
+
+void dcrypt_gnutls_init(struct module *module ATTR_UNUSED)
+{
+	gnutls_global_init();
+	dcrypt_set_vfs(&dcrypt_gnutls_vfs);
+}
+
+void dcrypt_gnutls_deinit(void)
+{
+	gnutls_global_deinit();
+}
diff --git a/src/lib-dcrypt/dcrypt-iostream-private.h b/src/lib-dcrypt/dcrypt-iostream-private.h
new file mode 100644
index 00000000000..d5cb7f17bca
--- /dev/null
+++ b/src/lib-dcrypt/dcrypt-iostream-private.h
@@ -0,0 +1,8 @@
+#ifndef DCRYPT_IOSTREAM_PRIVATE_H
+#define DCRYPT_IOSTREAM_PRIVATE 1
+
+static const unsigned char IOSTREAM_CRYPT_MAGIC[] = {'C','R','Y','P','T','E','D','\x03','\x07'};
+#define IOSTREAM_CRYPT_VERSION 2
+#define IOSTREAM_TAG_SIZE 16
+
+#endif
diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
new file mode 100644
index 00000000000..d60c5a4da23
--- /dev/null
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -0,0 +1,2015 @@
+#include "lib.h"
+#include "buffer.h"
+#include "str.h"
+#include "hex-binary.h"
+#include "safe-memset.h"
+#include "randgen.h"
+#include "array.h"
+#include "module-dir.h"
+#include <openssl/evp.h>
+#include <openssl/sha.h>
+#include <openssl/err.h>
+#include <openssl/rsa.h>
+#include <openssl/ec.h>
+#include <openssl/bio.h>
+#include <openssl/pem.h>
+#include <openssl/x509.h>
+#include <openssl/engine.h>
+#include <openssl/hmac.h>
+#include <openssl/objects.h>
+#include "dcrypt.h"
+#include "dcrypt-private.h"
+
+/**
+
+ key format documentation:
+ =========================
+
+ v1 key
+ ------
+ algo id = openssl NID
+ enctype = 0 = none, 1 = ecdhe, 2 = password
+ key id = sha256(hex encoded public point)
+
+ public key
+ ----------
+ 1<tab>algo id<tab>public point
+
+ private key
+ -----------
+ - enctype none
+ 1<tab>algo id<tab>0<tab>private point<tab>key id
+
+ - enctype ecdh (algorithm AES-256-CTR, key = SHA256(shared secret), IV = \0\0\0...)
+ 1<tab>algo id<tab>1<tab>private point<tab>ephemeral public key<tab>encryption key id<tab>key id
+
+ - enctype password (algorithm AES-256-CTR, key = PBKDF2(SHA1, 16, password, salt), IV = \0\0\0...)
+ 1<tab>algo id<tab>2<tab>private point<tab>salt<tab>key id
+
+ v2 key
+ ------
+ algo oid = ASN1 OID of key algorithm (RSA or EC curve)
+ enctype = 0 = none, 1 = ecdhe, 2 = password
+ key id = SHA256(i2d_PUBKEY)
+
+ public key
+ ----------
+ 2<tab>HEX(i2d_PUBKEY)
+
+ - enctype none
+ 2<tab>key algo oid<tab>0<tab>(RSA = i2d_PrivateKey, EC=Private Point)<tab>key id
+
+ - enctype ecdh, key,iv = PBKDF2(hash algo, rounds, shared secret, salt)
+ 2<tab>key algo oid<tab>1<tab>symmetric algo name<tab>salt<tab>hash algo<tab>rounds<tab>E(RSA = i2d_PrivateKey, EC=Private Point)<tab>ephemeral public key<tab>encryption key id<tab>key id
+
+ - enctype password, key,iv = PBKDF2(hash algo, rounds, password, salt)
+  2<tab>key algo oid<tab>1<tab>symmetric algo name<tab>salt<tab>hash algo<tab>rounds<tab>E(RSA = i2d_PrivateKey, EC=Private Point)<tab>key id
+**/
+
+struct dcrypt_context_symmetric {
+	pool_t pool;
+	const EVP_CIPHER *cipher;
+	EVP_CIPHER_CTX *ctx;
+	unsigned char *key;
+	unsigned char *iv;
+	unsigned char *aad;
+	size_t aad_len;
+	unsigned char *tag;
+	size_t tag_len;
+	int padding;
+	int mode;
+};
+
+struct dcrypt_context_hmac {
+	pool_t pool;
+	const EVP_MD *md;
+#if SSLEAY_VERSION_NUMBER >= 0x1010000fL
+	HMAC_CTX *ctx;
+#else
+	HMAC_CTX ctx;
+#endif
+	unsigned char *key;
+	size_t klen;
+};
+
+struct dcrypt_public_key {
+	void *ctx;
+};
+
+struct dcrypt_private_key {
+	void *ctx;
+};
+
+static
+bool dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key, const char **error_r);
+static
+bool dcrypt_openssl_public_key_id(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r);
+static
+bool dcrypt_openssl_public_key_id_old(struct dcrypt_public_key *key, buffer_t *result, const char **error_r);
+static
+bool dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r, const char **error_r ATTR_UNUSED);
+static
+void dcrypt_openssl_free_private_key(struct dcrypt_private_key **key);
+static
+void dcrypt_openssl_free_public_key(struct dcrypt_public_key **key);
+static
+bool dcrypt_openssl_rsa_decrypt(struct dcrypt_private_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r);
+
+static
+bool dcrypt_openssl_error(const char **error_r)
+{
+	if(error_r == NULL) return FALSE; /* caller is not really interested */
+	unsigned long ec = ERR_get_error();
+	*error_r = t_strdup_printf("%s", ERR_error_string(ec, NULL));
+	return FALSE;
+}
+
+/* legacy function for old formats that generates
+   hex encoded point from EC public key
+ */
+static
+char *ec_key_get_pub_point_hex(const EC_KEY *key)
+{
+	const EC_POINT *p;
+	const EC_GROUP *g;
+
+	p = EC_KEY_get0_public_key(key);
+	g = EC_KEY_get0_group(key);
+	return EC_POINT_point2hex(g, p, POINT_CONVERSION_COMPRESSED, NULL);
+}
+
+static
+bool dcrypt_openssl_ctx_sym_create(const char *algorithm, enum dcrypt_sym_mode mode, struct dcrypt_context_symmetric **ctx_r, const char **error_r)
+{
+	struct dcrypt_context_symmetric *ctx;
+	pool_t pool;
+	const EVP_CIPHER *cipher;
+	cipher = EVP_get_cipherbyname(algorithm);
+	if (cipher == NULL) {
+		if (error_r != NULL)
+			*error_r = t_strdup_printf("Invalid cipher %s", algorithm);
+		return FALSE;
+	}
+	/* allocate context */
+	pool = pool_alloconly_create("dcrypt openssl", 1024);
+	ctx = p_new(pool, struct dcrypt_context_symmetric, 1);
+	ctx->pool = pool;
+	ctx->cipher = cipher;
+	ctx->padding = 1;
+	ctx->mode =( mode == DCRYPT_MODE_ENCRYPT ? 1 : 0 );
+	*ctx_r = ctx;
+	return TRUE;
+}
+
+static
+void dcrypt_openssl_ctx_sym_destroy(struct dcrypt_context_symmetric **ctx)
+{
+	pool_t pool = (*ctx)->pool;
+	if ((*ctx)->ctx) EVP_CIPHER_CTX_free((*ctx)->ctx);
+	pool_unref(&pool);
+	*ctx = NULL;
+}
+
+static
+void dcrypt_openssl_ctx_sym_set_key(struct dcrypt_context_symmetric *ctx, const unsigned char *key, size_t key_len)
+{
+	if(ctx->key != NULL) p_free(ctx->pool, ctx->key);
+	ctx->key = p_malloc(ctx->pool, EVP_CIPHER_key_length(ctx->cipher));
+	memcpy(ctx->key, key, I_MIN(key_len,(size_t)EVP_CIPHER_key_length(ctx->cipher)));
+}
+
+static
+void dcrypt_openssl_ctx_sym_set_iv(struct dcrypt_context_symmetric *ctx, const unsigned char *iv, size_t iv_len)
+{
+	if(ctx->iv != NULL) p_free(ctx->pool, ctx->iv);
+	ctx->iv = p_malloc(ctx->pool, EVP_CIPHER_iv_length(ctx->cipher));
+	memcpy(ctx->iv, iv, I_MIN(iv_len,(size_t)EVP_CIPHER_iv_length(ctx->cipher)));
+}
+
+static
+void dcrypt_openssl_ctx_sym_set_key_iv_random(struct dcrypt_context_symmetric *ctx)
+{
+	if(ctx->key != NULL) p_free(ctx->pool, ctx->key);
+	if(ctx->iv != NULL) p_free(ctx->pool, ctx->iv);
+	ctx->key = p_malloc(ctx->pool, EVP_CIPHER_key_length(ctx->cipher));
+	random_fill(ctx->key, EVP_CIPHER_key_length(ctx->cipher));
+	ctx->iv = p_malloc(ctx->pool, EVP_CIPHER_iv_length(ctx->cipher));
+	random_fill(ctx->iv, EVP_CIPHER_iv_length(ctx->cipher));
+}
+
+static
+void dcrypt_openssl_ctx_sym_set_padding(struct dcrypt_context_symmetric *ctx, bool padding)
+{
+	ctx->padding = (padding?1:0);
+}
+
+static
+bool dcrypt_openssl_ctx_sym_get_key(struct dcrypt_context_symmetric *ctx, buffer_t *key)
+{
+	if(ctx->key == NULL) return FALSE;
+	buffer_append(key, ctx->key, EVP_CIPHER_key_length(ctx->cipher));
+	return TRUE;
+}
+static
+bool dcrypt_openssl_ctx_sym_get_iv(struct dcrypt_context_symmetric *ctx, buffer_t *iv)
+{
+	if(ctx->iv == NULL) return FALSE;
+	buffer_append(iv, ctx->iv, EVP_CIPHER_iv_length(ctx->cipher));
+	return TRUE;
+}
+
+static
+void dcrypt_openssl_ctx_sym_set_aad(struct dcrypt_context_symmetric *ctx, const unsigned char *aad, size_t aad_len)
+{
+	if (ctx->aad != NULL) p_free(ctx->pool, ctx->aad);
+	/* allow empty aad */
+	ctx->aad = p_malloc(ctx->pool, I_MAX(1,aad_len));
+	memcpy(ctx->aad, aad, aad_len);
+	ctx->aad_len = aad_len;
+}
+
+static
+bool dcrypt_openssl_ctx_sym_get_aad(struct dcrypt_context_symmetric *ctx, buffer_t *aad)
+{
+	if (ctx->aad == NULL) return FALSE;
+	buffer_append(aad, ctx->aad, ctx->aad_len);
+	return TRUE;
+}
+
+static
+void dcrypt_openssl_ctx_sym_set_tag(struct dcrypt_context_symmetric *ctx, const unsigned char *tag, size_t tag_len)
+{
+	if (ctx->tag != NULL) p_free(ctx->pool, ctx->tag);
+	/* unlike aad, tag cannot be empty */
+	ctx->tag = p_malloc(ctx->pool, tag_len);
+	memcpy(ctx->tag, tag, tag_len);
+	ctx->tag_len = tag_len;
+}
+
+static
+bool dcrypt_openssl_ctx_sym_get_tag(struct dcrypt_context_symmetric *ctx, buffer_t *tag)
+{
+	if (ctx->tag == NULL) return FALSE;
+	buffer_append(tag, ctx->tag, ctx->tag_len);
+	return TRUE;
+}
+
+static
+unsigned int dcrypt_openssl_ctx_sym_get_key_length(struct dcrypt_context_symmetric *ctx)
+{
+	return EVP_CIPHER_iv_length(ctx->cipher);
+}
+static
+unsigned int dcrypt_openssl_ctx_sym_get_iv_length(struct dcrypt_context_symmetric *ctx)
+{
+	return EVP_CIPHER_iv_length(ctx->cipher);
+}
+static
+unsigned int dcrypt_openssl_ctx_sym_get_block_size(struct dcrypt_context_symmetric *ctx)
+{
+	return EVP_CIPHER_block_size(ctx->cipher);
+}
+
+static
+bool dcrypt_openssl_ctx_sym_init(struct dcrypt_context_symmetric *ctx, const char **error_r)
+{
+	int ec;
+	int len;
+	i_assert(ctx->key != NULL);
+	i_assert(ctx->iv != NULL);
+	i_assert(ctx->ctx == NULL);
+
+	if((ctx->ctx = EVP_CIPHER_CTX_new()) == NULL)
+		return dcrypt_openssl_error(error_r);
+
+	ec = EVP_CipherInit_ex(ctx->ctx, ctx->cipher, NULL, ctx->key, ctx->iv, ctx->mode);
+	if (ec != 1) return dcrypt_openssl_error(error_r);
+	EVP_CIPHER_CTX_set_padding(ctx->ctx, ctx->padding);
+	len = 0;
+	if (ctx->aad != NULL) ec = EVP_CipherUpdate(ctx->ctx, NULL, &len, ctx->aad, ctx->aad_len);
+	if (ec != 1) return dcrypt_openssl_error(error_r);
+	return TRUE;
+}
+
+static
+bool dcrypt_openssl_ctx_sym_update(struct dcrypt_context_symmetric *ctx, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r)
+{
+	const size_t block_size = (size_t)EVP_CIPHER_block_size(ctx->cipher);
+	size_t buf_used = result->used;
+	unsigned char *buf;
+	int outl;
+
+	i_assert(ctx->ctx != NULL);
+
+	/* From `man 3 evp_cipherupdate`:
+
+	   EVP_EncryptUpdate() encrypts inl bytes from the buffer in and writes
+	   the encrypted version to out. This function can be called multiple
+	   times to encrypt successive blocks of data. The amount of data written
+	   depends on the block alignment of the encrypted data: as a result the
+	   amount of data written may be anything from zero bytes to
+	   (inl + cipher_block_size - 1) so out should contain sufficient room.
+	   The actual number of bytes written is placed in outl.
+	 */
+
+	buf = buffer_append_space_unsafe(result, data_len + block_size);
+	outl = 0;
+	if (EVP_CipherUpdate
+		(ctx->ctx, buf, &outl, data, data_len) != 1)
+		return dcrypt_openssl_error(error_r);
+	buffer_set_used_size(result, buf_used + outl);
+	return TRUE;
+}
+
+static
+bool dcrypt_openssl_ctx_sym_final(struct dcrypt_context_symmetric *ctx, buffer_t *result, const char **error_r)
+{
+	const size_t block_size = (size_t)EVP_CIPHER_block_size(ctx->cipher);
+	size_t buf_used = result->used;
+	unsigned char *buf;
+	int outl;
+	int ec;
+
+	i_assert(ctx->ctx != NULL);
+
+	/* From `man 3 evp_cipherupdate`:
+
+	   If padding is enabled (the default) then EVP_EncryptFinal_ex() encrypts
+	   the "final" data, that is any data that remains in a partial block. It
+	   uses standard block padding (aka PKCS padding). The encrypted final data
+	   is written to out which should have sufficient space for one cipher
+	   block. The number of bytes written is placed in outl. After this
+	   function is called the encryption operation is finished and no further
+	   calls to EVP_EncryptUpdate() should be made.
+	 */
+
+	buf = buffer_append_space_unsafe(result, block_size);
+	outl = 0;
+
+	/* when **DECRYPTING** set expected tag */
+	if (ctx->mode == 0 && ctx->tag != NULL) {
+		ec = EVP_CIPHER_CTX_ctrl(ctx->ctx, EVP_CTRL_GCM_SET_TAG, ctx->tag_len, ctx->tag);
+	} else ec = 1;
+
+	if (ec == 1)
+		ec = EVP_CipherFinal_ex(ctx->ctx, buf, &outl);
+
+	if (ec == 1) {
+		buffer_set_used_size(result, buf_used + outl);
+		/* when **ENCRYPTING** recover tag */
+		if (ctx->mode == 1 && ctx->aad != NULL) {
+			/* tag should be NULL here */
+			i_assert(ctx->tag == NULL);
+			/* openssl claims taglen is always 16, go figure .. */
+			ctx->tag = p_malloc(ctx->pool, EVP_GCM_TLS_TAG_LEN);
+			ec = EVP_CIPHER_CTX_ctrl(ctx->ctx, EVP_CTRL_GCM_GET_TAG, EVP_GCM_TLS_TAG_LEN, ctx->tag);
+			ctx->tag_len = EVP_GCM_TLS_TAG_LEN;
+		}
+	}
+
+	if (ec == 0 && error_r != NULL)
+		*error_r = "data authentication failed";
+	else if (ec < 0) dcrypt_openssl_error(error_r);
+
+	EVP_CIPHER_CTX_free(ctx->ctx);
+	ctx->ctx = NULL;
+
+	return ec == 1;
+}
+
+static
+bool dcrypt_openssl_ctx_hmac_create(const char *algorithm, struct dcrypt_context_hmac **ctx_r, const char **error_r)
+{
+	struct dcrypt_context_hmac *ctx;
+	pool_t pool;
+	const EVP_MD *md;
+	md = EVP_get_digestbyname(algorithm);
+	if(md == NULL) {
+		*error_r = t_strdup_printf("Invalid digest %s", algorithm);
+		return FALSE;
+	}
+	/* allocate context */
+	pool = pool_alloconly_create("dcrypt openssl", 1024);
+	ctx = p_new(pool, struct dcrypt_context_hmac, 1);
+	ctx->pool = pool;
+	ctx->md = md;
+	*ctx_r = ctx;
+	return TRUE;
+}
+
+static
+void dcrypt_openssl_ctx_hmac_destroy(struct dcrypt_context_hmac **ctx)
+{
+	pool_t pool = (*ctx)->pool;
+#if SSLEAY_VERSION_NUMBER >= 0x1010000fL
+	if ((*ctx)->ctx) HMAC_CTX_free((*ctx)->ctx);
+#else
+	HMAC_cleanup(&((*ctx)->ctx));
+#endif
+	pool_unref(&pool);
+	*ctx = NULL;
+}
+
+static
+void dcrypt_openssl_ctx_hmac_set_key(struct dcrypt_context_hmac *ctx, const unsigned char *key, size_t key_len)
+{
+	if(ctx->key != NULL) p_free(ctx->pool, ctx->key);
+	ctx->klen = I_MIN(key_len, HMAC_MAX_MD_CBLOCK);
+	ctx->key = p_malloc(ctx->pool, ctx->klen);
+	memcpy(ctx->key, key, ctx->klen);
+}
+static
+bool dcrypt_openssl_ctx_hmac_get_key(struct dcrypt_context_hmac *ctx, buffer_t *key)
+{
+	if(ctx->key == NULL) return FALSE;
+	buffer_append(key, ctx->key, ctx->klen);
+	return TRUE;
+}
+static
+void dcrypt_openssl_ctx_hmac_set_key_random(struct dcrypt_context_hmac *ctx)
+{
+	ctx->klen = HMAC_MAX_MD_CBLOCK;
+	ctx->key = p_malloc(ctx->pool, ctx->klen);
+	random_fill(ctx->key, ctx->klen);
+}
+
+static
+unsigned int dcrypt_openssl_ctx_hmac_get_digest_length(struct dcrypt_context_hmac *ctx)
+{
+	return EVP_MD_size(ctx->md);
+}
+
+static
+bool dcrypt_openssl_ctx_hmac_init(struct dcrypt_context_hmac *ctx, const char **error_r)
+{
+	int ec;
+	i_assert(ctx->md != NULL);
+#if SSLEAY_VERSION_NUMBER >= 0x1010000fL
+	ctx->ctx = HMAC_CTX_new();
+	if (ctx->ctx == NULL) return FALSE;
+	ec = HMAC_Init_ex(ctx->ctx, ctx->key, ctx->klen, ctx->md, NULL);
+#else
+	HMAC_CTX_init(&ctx->ctx);
+	ec = HMAC_Init_ex(&(ctx->ctx), ctx->key, ctx->klen, ctx->md, NULL);
+#endif
+	if (ec != 1) return dcrypt_openssl_error(error_r);
+	return TRUE;
+}
+static
+bool dcrypt_openssl_ctx_hmac_update(struct dcrypt_context_hmac *ctx, const unsigned char *data, size_t data_len, const char **error_r)
+{
+	int ec;
+#if SSLEAY_VERSION_NUMBER >= 0x1010000fL
+	ec = HMAC_Update(ctx->ctx, data, data_len);
+#else
+	ec = HMAC_Update(&(ctx->ctx), data, data_len);
+#endif
+	if (ec != 1) return dcrypt_openssl_error(error_r);
+	return TRUE;
+}
+static
+bool dcrypt_openssl_ctx_hmac_final(struct dcrypt_context_hmac *ctx, buffer_t *result, const char **error_r)
+{
+	int ec;
+	unsigned char buf[HMAC_MAX_MD_CBLOCK];
+	unsigned int outl;
+#if SSLEAY_VERSION_NUMBER >= 0x1010000fL
+	ec = HMAC_Final(ctx->ctx, buf, &outl);
+	HMAC_CTX_free(ctx->ctx);
+	ctx->ctx = NULL;
+#else
+	ec = HMAC_Final(&(ctx->ctx), buf, &outl);
+	HMAC_cleanup(&(ctx->ctx));
+#endif
+	if (ec == 1) {
+		buffer_append(result, buf, outl);
+	} else return dcrypt_openssl_error(error_r);
+	return TRUE;
+}
+
+static
+bool dcrypt_openssl_generate_ec_key(int nid, EVP_PKEY **key, const char **error_r)
+{
+	EVP_PKEY_CTX *pctx;
+	EVP_PKEY_CTX *ctx;
+	EVP_PKEY *params = NULL;
+
+	/* generate parameters for EC */
+	pctx = EVP_PKEY_CTX_new_id(EVP_PKEY_EC, NULL);
+	if (pctx == NULL ||
+	    EVP_PKEY_paramgen_init(pctx) < 1 ||
+	    EVP_PKEY_CTX_set_ec_paramgen_curve_nid(pctx, nid) < 1 ||
+	    EVP_PKEY_paramgen(pctx, &params) < 1)
+	{
+		dcrypt_openssl_error(error_r);
+		EVP_PKEY_CTX_free(pctx);
+		return FALSE;
+	}
+
+	/* generate key from parameters */
+	ctx = EVP_PKEY_CTX_new(params, NULL);
+	if (EVP_PKEY_keygen_init(ctx) < 1 ||
+	    EVP_PKEY_keygen(ctx, key) < 1)
+	{
+		dcrypt_openssl_error(error_r);
+		EVP_PKEY_free(params);
+		EVP_PKEY_CTX_free(pctx);
+		EVP_PKEY_CTX_free(ctx);
+		return FALSE;
+	}
+
+	EVP_PKEY_free(params);
+	EVP_PKEY_CTX_free(pctx);
+	EVP_PKEY_CTX_free(ctx);
+	EC_KEY_set_asn1_flag((*key)->pkey.ec, OPENSSL_EC_NAMED_CURVE);
+	EC_KEY_set_conv_form((*key)->pkey.ec, POINT_CONVERSION_COMPRESSED);
+	return TRUE;
+}
+
+static
+bool dcrypt_openssl_generate_rsa_key(int bits, EVP_PKEY **key, const char **error_r)
+{
+	int ec = 0;
+
+	EVP_PKEY_CTX *ctx;
+	ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, NULL);
+	if (ctx == NULL ||
+	    EVP_PKEY_keygen_init(ctx) < 1 ||
+	    EVP_PKEY_CTX_set_rsa_keygen_bits(ctx, bits) < 1 ||
+	    EVP_PKEY_keygen(ctx, key) < 1) {
+		dcrypt_openssl_error(error_r);
+		ec = -1;
+	}
+
+	EVP_PKEY_CTX_free(ctx);
+	return ec == 0;
+}
+
+static
+bool dcrypt_openssl_ecdh_derive_secret_local(struct dcrypt_private_key *local_key, buffer_t *R, buffer_t *S, const char **error_r)
+{
+	EVP_PKEY *local = (EVP_PKEY*)local_key;
+	BN_CTX *bn_ctx = BN_CTX_new();
+	const EC_GROUP *grp = EC_KEY_get0_group(local->pkey.ec);
+	EC_POINT *pub = EC_POINT_new(grp);
+	/* convert ephemeral key data EC point */
+	if (EC_POINT_oct2point(grp, pub, R->data, R->used, bn_ctx) != 1)
+	{
+		EC_POINT_free(pub);
+		BN_CTX_free(bn_ctx);
+		return dcrypt_openssl_error(error_r);
+	}
+	EC_KEY *ec_key = EC_KEY_new();
+	/* convert point to public key */
+	EC_KEY_set_conv_form(ec_key, POINT_CONVERSION_COMPRESSED);
+	EC_KEY_set_group(ec_key, grp);
+	EC_KEY_set_public_key(ec_key, pub);
+	EC_POINT_free(pub);
+	BN_CTX_free(bn_ctx);
+
+	/* make sure it looks like a valid key */
+	if (EC_KEY_check_key(ec_key) != 1) {
+		EC_KEY_free(ec_key);
+		return dcrypt_openssl_error(error_r);
+	}
+
+	EVP_PKEY *peer = EVP_PKEY_new();
+	EVP_PKEY_set1_EC_KEY(peer, ec_key);
+	EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new(local, NULL);
+
+	/* initialize derivation */
+	if (pctx == NULL ||
+	    EVP_PKEY_derive_init(pctx) != 1 ||
+	    EVP_PKEY_derive_set_peer(pctx, peer) != 1) {
+		EVP_PKEY_CTX_free(pctx);
+		EC_KEY_free(ec_key);
+		return dcrypt_openssl_error(error_r);
+	}
+
+	/* have to do it twice to get the data length */
+	size_t len;
+	if (EVP_PKEY_derive(pctx, NULL, &len) != 1) {
+		EVP_PKEY_CTX_free(pctx);
+		EC_KEY_free(ec_key);
+		return dcrypt_openssl_error(error_r);
+	}
+	unsigned char buf[len];
+	memset(buf,0,len);
+	if (EVP_PKEY_derive(pctx, buf, &len) != 1) {
+		EVP_PKEY_CTX_free(pctx);
+		EC_KEY_free(ec_key);
+		return dcrypt_openssl_error(error_r);
+	}
+	EVP_PKEY_CTX_free(pctx);
+	buffer_append(S, buf, len);
+	EC_KEY_free(ec_key);
+	EVP_PKEY_free(peer);
+	return TRUE;
+}
+
+static
+bool dcrypt_openssl_ecdh_derive_secret_peer(struct dcrypt_public_key *peer_key, buffer_t *R, buffer_t *S, const char **error_r)
+{
+	/* ensure peer_key is EC key */
+	EVP_PKEY *local = NULL;
+	EVP_PKEY *peer = (EVP_PKEY*)peer_key;
+	if (EVP_PKEY_base_id(peer) != EVP_PKEY_EC) {
+		if (error_r != NULL)
+			*error_r = "Only ECC key can be used";
+		return FALSE;
+	}
+
+	/* generate another key from same group */
+	int nid = EC_GROUP_get_curve_name(EC_KEY_get0_group(peer->pkey.ec));
+	if (!dcrypt_openssl_generate_ec_key(nid, &local, error_r)) return FALSE;
+
+	/* initialize */
+	EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new(local, NULL);
+	if (pctx == NULL ||
+	    EVP_PKEY_derive_init(pctx) != 1 ||
+	    EVP_PKEY_derive_set_peer(pctx, peer) != 1) {
+		EVP_PKEY_CTX_free(pctx);
+		return dcrypt_openssl_error(error_r);
+	}
+
+	/* derive */
+	size_t len;
+	if (EVP_PKEY_derive(pctx, NULL, &len) != 1) {
+		EVP_PKEY_CTX_free(pctx);
+		return dcrypt_openssl_error(error_r);
+	}
+	unsigned char buf[len];
+	if (EVP_PKEY_derive(pctx, buf, &len) != 1) {
+		EVP_PKEY_CTX_free(pctx);
+		return dcrypt_openssl_error(error_r);
+	}
+
+	EVP_PKEY_CTX_free(pctx);
+	buffer_append(S, buf, len);
+
+	/* get ephemeral key (=R) */
+	BN_CTX *bn_ctx = BN_CTX_new();
+	const EC_POINT *pub = EC_KEY_get0_public_key(local->pkey.ec);
+	const EC_GROUP *grp = EC_KEY_get0_group(local->pkey.ec);
+	len = EC_POINT_point2oct(grp, pub, POINT_CONVERSION_COMPRESSED, NULL, 0, bn_ctx);
+	unsigned char R_buf[len];
+	EC_POINT_point2oct(grp, pub, POINT_CONVERSION_COMPRESSED, R_buf, len, bn_ctx);
+	BN_CTX_free(bn_ctx);
+	buffer_append(R, R_buf, len);
+	EVP_PKEY_free(local);
+
+	return TRUE;
+}
+
+static
+bool dcrypt_openssl_pbkdf2(const unsigned char *password, size_t password_len, const unsigned char *salt, size_t salt_len,
+	const char *hash, unsigned int rounds, buffer_t *result, unsigned int result_len, const char **error_r)
+{
+	int ret;
+	i_assert(rounds > 0);
+	i_assert(result_len > 0);
+	i_assert(result != NULL);
+	T_BEGIN {
+		/* determine MD */
+		const EVP_MD* md = EVP_get_digestbyname(hash);
+		if (md == NULL) {
+			*error_r = t_strdup_printf("Invalid digest %s", hash);
+			return FALSE;
+		}
+
+		unsigned char buffer[result_len];
+		if ((ret = PKCS5_PBKDF2_HMAC((const char*)password, password_len, salt, salt_len, rounds,
+						md, result_len, buffer)) == 1) {
+			buffer_append(result, buffer, result_len);
+		}
+	} T_END;
+	if (ret != 1) return dcrypt_openssl_error(error_r);
+	return TRUE;
+}
+
+static
+bool dcrypt_openssl_generate_keypair(struct dcrypt_keypair *pair_r, enum dcrypt_key_type kind, unsigned int bits, const char *curve, const char **error_r)
+{
+	if (kind == DCRYPT_KEY_RSA) {
+		if (dcrypt_openssl_generate_rsa_key(bits, (EVP_PKEY**)&(pair_r->priv), error_r) == 0) {
+			return dcrypt_openssl_private_to_public_key(pair_r->priv, &(pair_r->pub), error_r);
+		} else return dcrypt_openssl_error(error_r);
+	} else if (kind == DCRYPT_KEY_EC) {
+		int nid = OBJ_sn2nid(curve);
+		if (nid == NID_undef) {
+			if (error_r != NULL)
+				*error_r = t_strdup_printf("Unknown EC curve %s", curve);
+			return FALSE;
+		}
+		if (dcrypt_openssl_generate_ec_key(nid, (EVP_PKEY**)&(pair_r->priv), error_r) == 0) {
+			return dcrypt_openssl_private_to_public_key(pair_r->priv, &(pair_r->pub), error_r);
+		} else return dcrypt_openssl_error(error_r);
+	}
+	if (error_r != NULL)
+		*error_r = "Key type not supported in this build";
+	return FALSE;
+}
+
+static
+bool dcrypt_openssl_decrypt_point_v1(buffer_t *data, buffer_t *key, BIGNUM **point_r, const char **error_r)
+{
+	struct dcrypt_context_symmetric *dctx;
+	buffer_t *tmp = buffer_create_dynamic(pool_datastack_create(), 64);
+
+	if (!dcrypt_openssl_ctx_sym_create("aes-256-ctr", DCRYPT_MODE_DECRYPT, &dctx, error_r)) {
+		return FALSE;
+	}
+
+	/* v1 KEYS have all-zero IV - have to use it ourselves too */
+	dcrypt_openssl_ctx_sym_set_iv(dctx, (const unsigned char*)"\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0", 16);
+	dcrypt_openssl_ctx_sym_set_key(dctx, key->data, key->used);
+
+	if (!dcrypt_openssl_ctx_sym_init(dctx, error_r) ||
+	    !dcrypt_openssl_ctx_sym_update(dctx, data->data, data->used, tmp, error_r) ||
+	    !dcrypt_openssl_ctx_sym_final(dctx, tmp, error_r)) {
+		dcrypt_openssl_ctx_sym_destroy(&dctx);
+		return FALSE;
+	}
+
+	dcrypt_openssl_ctx_sym_destroy(&dctx);
+
+	*point_r = BN_bin2bn(tmp->data, tmp->used, NULL);
+	safe_memset(buffer_get_modifiable_data(tmp, NULL), 0,tmp->used);
+	buffer_set_used_size(key, 0);
+
+	if (*point_r == FALSE)
+		return dcrypt_openssl_error(error_r);
+
+	return TRUE;
+}
+
+static
+bool dcrypt_openssl_decrypt_point_ec_v1(struct dcrypt_private_key *dec_key,
+	const char *data_hex, const char *peer_key_hex, BIGNUM **point_r, const char **error_r)
+{
+	buffer_t *peer_key, *data, key, *secret;
+	bool res;
+
+	data = buffer_create_dynamic(pool_datastack_create(), 128);
+	peer_key = buffer_create_dynamic(pool_datastack_create(), 64);
+
+	hex_to_binary(data_hex, data);
+	hex_to_binary(peer_key_hex, peer_key);
+
+	secret = buffer_create_dynamic(pool_datastack_create(), 64);
+
+	if (!dcrypt_openssl_ecdh_derive_secret_local(dec_key, peer_key, secret, error_r))
+		return FALSE;
+
+	/* run it thru SHA256 once */
+	unsigned char digest[SHA256_DIGEST_LENGTH];
+	SHA256(secret->data, secret->used, digest);
+	safe_memset(buffer_get_modifiable_data(secret, NULL), 0, secret->used);
+	buffer_set_used_size(secret, 0);
+	buffer_create_from_const_data(&key, digest, SHA256_DIGEST_LENGTH);
+
+	/* then use this as key */
+	res = dcrypt_openssl_decrypt_point_v1(data, &key, point_r, error_r);
+	memset(digest, 0, sizeof(digest));
+	safe_memset(digest, 0, SHA256_DIGEST_LENGTH);
+
+	return res;
+}
+
+static
+bool dcrypt_openssl_decrypt_point_password_v1(const char *data_hex, const char *password_hex,
+	const char *salt_hex, BIGNUM **point_r, const char **error_r)
+{
+	buffer_t *salt, *data, *password, *key;
+	struct dcrypt_context_symmetric *dctx;
+
+	data = buffer_create_dynamic(pool_datastack_create(), 128);
+	salt = buffer_create_dynamic(pool_datastack_create(), 16);
+	password = buffer_create_dynamic(pool_datastack_create(), 32);
+	key = buffer_create_dynamic(pool_datastack_create(), 32);
+
+	hex_to_binary(data_hex, data);
+	hex_to_binary(salt_hex, salt);
+	hex_to_binary(password_hex, password);
+
+	/* aes-256-ctr uses 32 byte key, and v1 uses all-zero IV */
+	if (!dcrypt_openssl_pbkdf2(password->data, password->used, salt->data, salt->used,
+				   "sha256", 16, key, 32, error_r)) {
+		dcrypt_ctx_sym_destroy(&dctx);
+		return FALSE;
+	}
+
+	return dcrypt_openssl_decrypt_point_v1(data, key, point_r, error_r);
+}
+
+static
+bool dcrypt_openssl_load_private_key_dovecot_v1(struct dcrypt_private_key **key_r,
+	int len, const char **input, const char *password, struct dcrypt_private_key *dec_key,
+	const char **error_r)
+{
+	int nid, ec, enctype;
+	EC_KEY *eckey = NULL;
+	BIGNUM *point = NULL;
+
+	if (str_to_int(input[1], &nid) != 0) {
+		if (error_r != NULL)
+			*error_r = "Corrupted data";
+		return FALSE;
+	}
+
+	if (str_to_int(input[2], &enctype) != 0) {
+		if (error_r != NULL)
+			*error_r = "Corrupted data";
+		return FALSE;
+	}
+
+	eckey = EC_KEY_new_by_curve_name(nid);
+	if (eckey == NULL) return dcrypt_openssl_error(error_r);
+
+	/* decode and optionally decipher private key value */
+	if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_NONE) {
+		point = BN_new();
+		if (BN_hex2bn(&point, input[3]) < 1) {
+			BN_free(point);
+			EC_KEY_free(eckey);
+			return dcrypt_openssl_error(error_r);
+		}
+	} else if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PASSWORD) {
+		/* by password */
+		const char *enc_priv_pt = input[3];
+		const char *salt = input[4];
+		if (!dcrypt_openssl_decrypt_point_password_v1(enc_priv_pt, password, salt, &point, error_r)) {
+			EC_KEY_free(eckey);
+			return FALSE;
+		}
+	} else if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PK) {
+		/* by key */
+		const char *enc_priv_pt = input[3];
+		const char *peer_key = input[4];
+		if (!dcrypt_openssl_decrypt_point_ec_v1(dec_key, enc_priv_pt, peer_key, &point, error_r)) {
+			EC_KEY_free(eckey);
+			return FALSE;
+		}
+	} else {
+		if (error_r != NULL)
+			*error_r = "Invalid key data";
+		EC_KEY_free(eckey);
+		return FALSE;
+	}
+
+	/* assign private key */
+	BN_CTX *bnctx = BN_CTX_new();
+	EC_KEY_set_conv_form(eckey, POINT_CONVERSION_COMPRESSED);
+	EC_KEY_set_private_key(eckey, point);
+	EC_KEY_precompute_mult(eckey, bnctx);
+	EC_KEY_set_asn1_flag(eckey, OPENSSL_EC_NAMED_CURVE);
+	EC_POINT *pub = EC_POINT_new(EC_KEY_get0_group(eckey));
+	/* calculate public key */
+	ec = EC_POINT_mul(EC_KEY_get0_group(eckey), pub, point, NULL, NULL, bnctx);
+	EC_KEY_set_public_key(eckey, pub);
+	BN_free(point);
+	EC_POINT_free(pub);
+	BN_CTX_free(bnctx);
+
+	/* make sure it looks OK and is correct */
+	if (ec == 1 && EC_KEY_check_key(eckey) == 1) {
+		unsigned char digest[SHA256_DIGEST_LENGTH];
+		/* validate that the key was loaded correctly */
+		char *id = ec_key_get_pub_point_hex(eckey);
+		SHA256((unsigned char*)id, strlen(id), digest);
+		OPENSSL_free(id);
+		const char *digest_hex = binary_to_hex(digest, SHA256_DIGEST_LENGTH);
+		if (strcmp(digest_hex, input[len-1]) != 0) {
+			if (error_r != NULL)
+				*error_r = "Key id mismatch after load";
+			EC_KEY_free(eckey);
+			return FALSE;
+		}
+		EVP_PKEY *key = EVP_PKEY_new();
+		EVP_PKEY_set1_EC_KEY(key, eckey);
+		EC_KEY_free(eckey);
+		*key_r = (struct dcrypt_private_key *)key;
+		return TRUE;
+	}
+
+	EC_KEY_free(eckey);
+
+	return dcrypt_openssl_error(error_r);
+}
+
+/* encrypt/decrypt private keys */
+static
+bool dcrypt_openssl_cipher_key_dovecot_v2(const char *cipher, enum dcrypt_sym_mode mode,
+	buffer_t *input, buffer_t *secret, buffer_t *salt, const char *digalgo, unsigned int rounds,
+	buffer_t *result_r, const char **error_r)
+{
+	struct dcrypt_context_symmetric *dctx;
+	bool res;
+
+	if (!dcrypt_openssl_ctx_sym_create(cipher, mode, &dctx, error_r)) {
+		return FALSE;
+	}
+
+	/* generate encryption key/iv based on secret/salt */
+	buffer_t *key_data = buffer_create_dynamic(pool_datastack_create(), 128);
+	res = dcrypt_openssl_pbkdf2(secret->data, secret->used, salt->data, salt->used,
+		digalgo, rounds, key_data,
+		dcrypt_openssl_ctx_sym_get_key_length(dctx)+dcrypt_openssl_ctx_sym_get_iv_length(dctx), error_r);
+
+	if (!res) {
+		dcrypt_openssl_ctx_sym_destroy(&dctx);
+		return FALSE;
+	}
+
+	buffer_t *tmp = buffer_create_dynamic(pool_datastack_create(), 128);
+	const unsigned char *kd = buffer_free_without_data(&key_data);
+
+	/* perform ciphering */
+	dcrypt_openssl_ctx_sym_set_key(dctx, kd, dcrypt_openssl_ctx_sym_get_key_length(dctx));
+	dcrypt_openssl_ctx_sym_set_iv(dctx, kd+dcrypt_openssl_ctx_sym_get_key_length(dctx), dcrypt_openssl_ctx_sym_get_iv_length(dctx));
+
+	if (!dcrypt_openssl_ctx_sym_init(dctx, error_r) ||
+	    !dcrypt_openssl_ctx_sym_update(dctx, input->data, input->used, tmp, error_r) ||
+	    !dcrypt_openssl_ctx_sym_final(dctx, tmp, error_r)) {
+		res = FALSE;
+	} else {
+		/* provide result if succeeded */
+		buffer_append_buf(result_r, tmp, 0, (size_t)-1);
+		res = TRUE;
+	}
+	/* and ensure no data leaks */
+	safe_memset(buffer_get_modifiable_data(tmp, NULL), 0, tmp->used);
+
+	dcrypt_openssl_ctx_sym_destroy(&dctx);
+	return res;
+}
+
+static
+bool dcrypt_openssl_load_private_key_dovecot_v2(struct dcrypt_private_key **key_r,
+	int len, const char **input, const char *password, struct dcrypt_private_key *dec_key,
+	const char **error_r)
+{
+	int enctype;
+	buffer_t *key_data = buffer_create_dynamic(pool_datastack_create(), 256);
+
+	/* check for encryption type */
+	if (str_to_int(input[2], &enctype) != 0) {
+		if (error_r != NULL)
+			*error_r = "Corrupted data";
+		return FALSE;
+	}
+
+	if (enctype < 0 || enctype > 2) {
+		if (error_r != NULL)
+			*error_r = "Corrupted data";
+		return FALSE;
+	}
+
+	/* match encryption type to field counts */
+	if ((enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_NONE && len != 5) ||
+	    (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PASSWORD && len != 9) ||
+ 	    (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PK && len != 11)) {
+		if (error_r != NULL)
+			*error_r = "Corrupted data";
+		return FALSE;
+	}
+
+	/* get key type */
+	int nid = OBJ_txt2nid(input[1]);
+
+	if (nid == NID_undef)
+		return dcrypt_openssl_error(error_r);
+
+	/* decode and possibly decipher private key value */
+	if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_NONE) {
+		if (hex_to_binary(input[3], key_data) != 0) {
+			if (error_r != NULL)
+				*error_r = "Corrupted data";
+		}
+	} else if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PK) {
+		unsigned int rounds;
+		struct dcrypt_public_key *pubkey = NULL;
+		if (str_to_uint(input[6], &rounds) != 0) {
+			if (error_r != NULL)
+				*error_r = "Corrupted data";
+			return FALSE;
+		}
+
+		buffer_t *data = buffer_create_dynamic(pool_datastack_create(), 128);
+
+		/* check that we have correct decryption key */
+		if (!dcrypt_openssl_private_to_public_key(dec_key, &pubkey, error_r) ||
+		    !dcrypt_openssl_public_key_id(pubkey, "sha256", data, error_r)) {
+			if (pubkey != NULL) dcrypt_openssl_free_public_key(&pubkey);
+			return FALSE;
+		}
+
+		dcrypt_openssl_free_public_key(&pubkey);
+
+		if (strcmp(binary_to_hex(data->data, data->used), input[9]) != 0) {
+			dcrypt_openssl_free_public_key(&pubkey);
+			if (error_r != NULL)
+				*error_r = "No private key available";
+			return FALSE;
+		}
+
+
+		buffer_t *salt, *peer_key, *secret;
+		salt = buffer_create_dynamic(pool_datastack_create(), strlen(input[4])/2);
+		peer_key = buffer_create_dynamic(pool_datastack_create(), strlen(input[8])/2);
+		secret = buffer_create_dynamic(pool_datastack_create(), 128);
+
+		buffer_set_used_size(data, 0);
+		hex_to_binary(input[4], salt);
+		hex_to_binary(input[8], peer_key);
+		hex_to_binary(input[7], data);
+
+		/* get us secret value to use for key/iv generation */
+		if (EVP_PKEY_base_id((EVP_PKEY*)dec_key) == EVP_PKEY_RSA) {
+			if (!dcrypt_openssl_rsa_decrypt(dec_key, peer_key->data, peer_key->used, secret, error_r))
+				return FALSE;
+		} else {
+			/* perform ECDH */
+			if (!dcrypt_openssl_ecdh_derive_secret_local(dec_key, peer_key, secret, error_r))
+				return FALSE;
+		}
+		/* decrypt key */
+		if (!dcrypt_openssl_cipher_key_dovecot_v2(input[3], DCRYPT_MODE_DECRYPT, data, secret, salt,
+		    input[5], rounds, key_data, error_r)) {
+			return FALSE;
+		}
+	} else if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PASSWORD) {
+		unsigned int rounds;
+		if (str_to_uint(input[6], &rounds) != 0) {
+			if (error_r != NULL)
+				*error_r = "Corrupted data";
+			return FALSE;
+		}
+
+		buffer_t *salt, secret, *data;
+		salt = buffer_create_dynamic(pool_datastack_create(), strlen(input[4])/2);
+		buffer_create_from_const_data(&secret, password, strlen(password));
+		data = buffer_create_dynamic(pool_datastack_create(), strlen(input[7])/2);
+		if (hex_to_binary(input[4], salt) != 0 ||
+		    hex_to_binary(input[7], data) != 0) {
+			if (error_r != NULL)
+				*error_r = "Corrupted data";
+			return FALSE;
+		}
+
+		if (!dcrypt_openssl_cipher_key_dovecot_v2(input[3], DCRYPT_MODE_DECRYPT, data, &secret, salt,
+		    input[5], rounds, key_data, error_r)) {
+			return FALSE;
+		}
+	}
+
+	/* decode actual key */
+	if (EVP_PKEY_type(nid) == EVP_PKEY_RSA) {
+		RSA *rsa = RSA_new();
+		const unsigned char *ptr = buffer_get_data(key_data, NULL);
+		if (d2i_RSAPrivateKey(&rsa, &ptr, key_data->used) == NULL ||
+		    RSA_check_key(rsa) != 1) {
+			safe_memset(buffer_get_modifiable_data(key_data, NULL), 0, key_data->used);
+			RSA_free(rsa);
+			return dcrypt_openssl_error(error_r);
+		}
+		safe_memset(buffer_get_modifiable_data(key_data, NULL), 0, key_data->used);
+		buffer_set_used_size(key_data, 0);
+		EVP_PKEY *pkey = EVP_PKEY_new();
+		EVP_PKEY_set1_RSA(pkey, rsa);
+		*key_r = (struct dcrypt_private_key *)pkey;
+	} else {
+		int ec;
+		BIGNUM *point = BN_new();
+		if (BN_mpi2bn(key_data->data, key_data->used, point) == NULL) {
+			safe_memset(buffer_get_modifiable_data(key_data, NULL), 0, key_data->used);
+			BN_free(point);
+			return dcrypt_openssl_error(error_r);
+		}
+		EC_KEY *eckey = EC_KEY_new_by_curve_name(nid);
+		safe_memset(buffer_get_modifiable_data(key_data, NULL), 0, key_data->used);
+		buffer_set_used_size(key_data, 0);
+		if (eckey == NULL) {
+			return dcrypt_openssl_error(error_r);
+		}
+		BN_CTX *bnctx = BN_CTX_new();
+		EC_KEY_set_conv_form(eckey, POINT_CONVERSION_COMPRESSED);
+		EC_KEY_set_private_key(eckey, point);
+		EC_KEY_precompute_mult(eckey, bnctx);
+		EC_KEY_set_asn1_flag(eckey, OPENSSL_EC_NAMED_CURVE);
+		EC_POINT *pub = EC_POINT_new(EC_KEY_get0_group(eckey));
+		/* calculate public key */
+		ec = EC_POINT_mul(EC_KEY_get0_group(eckey), pub, point, NULL, NULL, bnctx);
+		EC_KEY_set_public_key(eckey, pub);
+		BN_free(point);
+		EC_POINT_free(pub);
+		BN_CTX_free(bnctx);
+		/* make sure the EC key is valid */
+		if (ec == 1 && EC_KEY_check_key(eckey) == 1) {
+			EVP_PKEY *key = EVP_PKEY_new();
+			EVP_PKEY_set1_EC_KEY(key, eckey);
+			EC_KEY_free(eckey);
+			*key_r = (struct dcrypt_private_key *)key;
+		} else {
+			EC_KEY_free(eckey);
+			return dcrypt_openssl_error(error_r);
+		}
+	}
+
+	/* finally compare key to key id */
+	struct dcrypt_public_key *pubkey = NULL;
+	dcrypt_openssl_private_to_public_key(*key_r, &pubkey, NULL);
+	dcrypt_openssl_public_key_id(pubkey, "sha256", key_data, NULL);
+	dcrypt_openssl_free_public_key(&pubkey);
+
+	if (strcmp(binary_to_hex(key_data->data, key_data->used), input[len-1]) != 0) {
+		dcrypt_openssl_free_private_key(key_r);
+		if (error_r != NULL)
+			*error_r = "Key id mismatch after load";
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+
+static
+bool dcrypt_openssl_load_private_key_dovecot(struct dcrypt_private_key **key_r,
+	const char *data, const char *password, struct dcrypt_private_key *key,
+	const char **error_r)
+{
+	bool ret;
+	T_BEGIN {
+		const char **input = t_strsplit_tab(data);
+		size_t len;
+		for(len=0;input[len]!=NULL;len++);
+		if (len < 4) {
+			if (error_r != NULL)
+				*error_r = "Corrupted data";
+			ret = FALSE;
+		} else if (*(input[0])== '1')
+			ret = dcrypt_openssl_load_private_key_dovecot_v1(key_r, len, input, password, key, error_r);
+		else if (*(input[0])== '2')
+			ret = dcrypt_openssl_load_private_key_dovecot_v2(key_r, len, input, password, key, error_r);
+		else {
+			if (error_r != NULL)
+				*error_r = "Unsupported key version";
+			ret = FALSE;
+		}
+	} T_END;
+	return ret;
+}
+
+static
+int dcrypt_openssl_load_public_key_dovecot_v1(struct dcrypt_public_key **key_r,
+	int len, const char **input, const char **error_r)
+{
+	int nid;
+	if (len != 3) {
+		if (error_r != NULL)
+			*error_r = "Corrupted data";
+		return -1;
+	}
+	if (str_to_int(input[1], &nid) != 0) {
+		if (error_r != NULL)
+			*error_r = "Corrupted data";
+		return -1;
+	}
+
+	EC_KEY *eckey = EC_KEY_new_by_curve_name(nid);
+	if (eckey == NULL) {
+		dcrypt_openssl_error(error_r);
+		return -1;
+	}
+
+	EC_KEY_set_asn1_flag(eckey, OPENSSL_EC_NAMED_CURVE);
+	BN_CTX *bnctx = BN_CTX_new();
+
+	EC_POINT *point = EC_POINT_new(EC_KEY_get0_group(eckey));
+	if (EC_POINT_hex2point(EC_KEY_get0_group(eckey),
+ 	    input[2], point, bnctx) == NULL) {
+		BN_CTX_free(bnctx);
+		EC_KEY_free(eckey);
+		EC_POINT_free(point);
+		dcrypt_openssl_error(error_r);
+		return -1;
+	}
+	BN_CTX_free(bnctx);
+
+	EC_KEY_set_public_key(eckey, point);
+	EC_KEY_set_asn1_flag(eckey, OPENSSL_EC_NAMED_CURVE);
+
+	EC_POINT_free(point);
+
+	if (EC_KEY_check_key(eckey) == 1) {
+		EVP_PKEY *key = EVP_PKEY_new();
+		EVP_PKEY_set1_EC_KEY(key, eckey);
+		*key_r = (struct dcrypt_public_key *)key;
+		return 0;
+	}
+
+	dcrypt_openssl_error(error_r);
+	return -1;
+}
+
+static
+bool dcrypt_openssl_load_public_key_dovecot_v2(struct dcrypt_public_key **key_r,
+	int len, const char **input, const char **error_r)
+{
+	if (len != 2 || strlen(input[1]) < 2 || (strlen(input[1])%2) != 0) {
+		if (error_r != NULL)
+			*error_r = "Corrupted data";
+		return -1;
+	}
+	buffer_t tmp;
+	size_t keylen = strlen(input[1])/2;
+	unsigned char keybuf[keylen];
+	buffer_create_from_data(&tmp, keybuf, keylen);
+	hex_to_binary(input[1], &tmp);
+
+	EVP_PKEY *pkey = EVP_PKEY_new();
+	if (d2i_PUBKEY(&pkey, (const unsigned char**)(tmp.data), tmp.used)==NULL) {
+		EVP_PKEY_free(pkey);
+		dcrypt_openssl_error(error_r);
+		return -1;
+	}
+
+	*key_r = (struct dcrypt_public_key *)pkey;
+	return 0;
+}
+
+static
+bool dcrypt_openssl_load_public_key_dovecot(struct dcrypt_public_key **key_r,
+	const char *data, const char **error_r)
+{
+	int ec = 0;
+
+	T_BEGIN {
+		const char **input = t_strsplit_tab(data);
+		size_t len;
+		for(len=0;input[len]!=NULL;len++);
+		if (len < 2) ec = -1;
+		if (ec == 0 && *(input[0]) == '1') {
+			ec = dcrypt_openssl_load_public_key_dovecot_v1(key_r, len,
+				input, error_r);
+		} else if (ec == 0 && *(input[0]) == '2') {
+			ec = dcrypt_openssl_load_public_key_dovecot_v2(key_r, len,
+				input, error_r);
+		} else {
+			if (error_r != NULL)
+				*error_r = "Unsupported key version";
+			ec = -1;
+		}
+	} T_END;
+
+	return (ec == 0 ? TRUE : FALSE);
+}
+
+static
+bool dcrypt_openssl_encrypt_private_key_dovecot(buffer_t *key, int enctype, const char *cipher, const char *password,
+	struct dcrypt_public_key *enc_key, buffer_t *destination, const char **error_r)
+{
+	bool res;
+	unsigned char *ptr;
+
+	unsigned char salt[8];
+	buffer_t *peer_key = buffer_create_dynamic(pool_datastack_create(), 128);
+	buffer_t *secret = buffer_create_dynamic(pool_datastack_create(), 128);
+	cipher = t_str_lcase(cipher);
+
+	str_append(destination, cipher);
+	str_append_c(destination, '\t');
+	random_fill(salt, sizeof(salt));
+	binary_to_hex_append(destination, salt, sizeof(salt));
+	buffer_t saltbuf;
+	buffer_create_from_const_data(&saltbuf, salt, sizeof(salt));
+
+	/* so we don't have to make new version if we ever upgrade these */
+	str_append(destination, t_strdup_printf("\t%s\t%d\t",
+		DCRYPT_DOVECOT_KEY_ENCRYPT_HASH,
+		DCRYPT_DOVECOT_KEY_ENCRYPT_ROUNDS));
+
+	if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PK) {
+		if (EVP_PKEY_base_id((EVP_PKEY*)enc_key) == EVP_PKEY_RSA) {
+			size_t used = buffer_get_used_size(secret);
+			/* peer key, in this case, is encrypted secret, which is 16 bytes of data */
+			ptr = buffer_append_space_unsafe(secret, 16);
+			random_fill(ptr, 16);
+			buffer_set_used_size(secret, used+16);
+			if (!dcrypt_rsa_encrypt(enc_key, secret->data, secret->used, peer_key, error_r)) {
+				return FALSE;
+			}
+		} else if (EVP_PKEY_base_id((EVP_PKEY*)enc_key) == EVP_PKEY_EC) {
+			/* generate secret by ECDHE */
+			if (!dcrypt_openssl_ecdh_derive_secret_peer(enc_key, peer_key, secret, error_r)) {
+				return FALSE;
+			}
+		} else {
+			if (error_r != NULL)
+				*error_r = "Unsupported encryption key";
+			return FALSE;
+		}
+		/* add encryption key id, reuse peer_key buffer */
+	} else if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PASSWORD) {
+		str_append(secret, password);
+	}
+
+	/* encrypt key using secret and salt */
+	buffer_t *tmp = buffer_create_dynamic(pool_datastack_create(), 128);
+	res = dcrypt_openssl_cipher_key_dovecot_v2(cipher, DCRYPT_MODE_ENCRYPT, key, secret, &saltbuf,
+		DCRYPT_DOVECOT_KEY_ENCRYPT_HASH, DCRYPT_DOVECOT_KEY_ENCRYPT_ROUNDS, tmp, error_r);
+	safe_memset(buffer_get_modifiable_data(secret, NULL), 0, secret->used);
+	binary_to_hex_append(destination, tmp->data, tmp->used);
+
+	/* some additional fields or private key version */
+	if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PK) {
+		str_append_c(destination, '\t');
+
+		/* for RSA, this is the actual encrypted secret */
+		binary_to_hex_append(destination, peer_key->data, peer_key->used);
+		str_append_c(destination, '\t');
+
+		buffer_set_used_size(peer_key, 0);
+		if (!dcrypt_openssl_public_key_id(enc_key, "sha256", peer_key, error_r))
+			return FALSE;
+		binary_to_hex_append(destination, peer_key->data, peer_key->used);
+	}
+	return res;
+}
+
+static
+bool dcrypt_openssl_store_private_key_dovecot(struct dcrypt_private_key *key, const char *cipher, buffer_t *destination,
+	const char *password, struct dcrypt_public_key *enc_key, const char **error_r)
+{
+	size_t dest_used = buffer_get_used_size(destination);
+	const char *cipher2 = NULL;
+	EVP_PKEY *pkey = (EVP_PKEY*)key;
+	char objtxt[80]; /* openssl manual says this is OK */
+	ASN1_OBJECT *obj;
+	if (EVP_PKEY_base_id(pkey) == EVP_PKEY_EC) {
+		/* because otherwise we get wrong nid */
+		obj = OBJ_nid2obj(EC_GROUP_get_curve_name(EC_KEY_get0_group(pkey->pkey.ec)));
+
+	} else {
+		obj = OBJ_nid2obj(EVP_PKEY_id(pkey));
+	}
+
+	int enctype = 0;
+	int ln = OBJ_obj2txt(objtxt, sizeof(objtxt), obj, 1);
+	if (ln < 1)
+		return dcrypt_openssl_error(error_r);
+	if (ln > (int)sizeof(objtxt)) {
+		if (error_r != NULL)
+			*error_r = "Object identifier too long";
+		return FALSE;
+	}
+
+	buffer_t *buf = buffer_create_dynamic(pool_datastack_create(), 256);
+
+	/* convert key to private key value */
+	if (EVP_PKEY_base_id(pkey) == EVP_PKEY_RSA) {
+		unsigned char *ptr;
+		RSA *rsa = pkey->pkey.rsa;
+		int ln = i2d_RSAPrivateKey(rsa, &ptr);
+		if (ln < 1)
+			return dcrypt_openssl_error(error_r);
+		buffer_append(buf, ptr, ln);
+	} else if (EVP_PKEY_base_id(pkey) == EVP_PKEY_EC) {
+		unsigned char *ptr;
+		EC_KEY *eckey = pkey->pkey.ec;
+		const BIGNUM *pk = EC_KEY_get0_private_key(eckey);
+		/* serialize to MPI which is portable */
+		int len = BN_bn2mpi(pk, NULL);
+		ptr = buffer_append_space_unsafe(buf, len);
+		BN_bn2mpi(pk, ptr);
+	} else {
+		if (*error_r != NULL)
+			*error_r = "Unsupported key type";
+		return FALSE;
+	}
+
+	/* see if we want ECDH based or password based encryption */
+	if (cipher != NULL && strncasecmp(cipher, "ecdh-", 5) == 0) {
+		i_assert(enc_key != NULL);
+		i_assert(password == NULL);
+		enctype = DCRYPT_DOVECOT_KEY_ENCRYPT_PK;
+		cipher2 = cipher+5;
+	} else if (cipher != NULL) {
+		i_assert(enc_key == NULL);
+		i_assert(password != NULL);
+		enctype = DCRYPT_DOVECOT_KEY_ENCRYPT_PASSWORD;
+		cipher2 = cipher;
+	}
+
+	/* put in OID and encryption type */
+	str_append(destination, t_strdup_printf("2\t%s\t%d\t",
+		objtxt, enctype));
+
+	/* perform encryption if desired */
+	if (enctype > 0) {
+		if (!dcrypt_openssl_encrypt_private_key_dovecot(buf, enctype, cipher2, password, enc_key, destination, error_r)) {
+			buffer_set_used_size(destination, dest_used);
+			return FALSE;
+		}
+	} else {
+		binary_to_hex_append(destination, buf->data, buf->used);
+	}
+
+	/* append public key id */
+	struct dcrypt_public_key *pubkey = NULL;
+	if (!dcrypt_openssl_private_to_public_key(key, &pubkey, error_r)) {
+		buffer_set_used_size(destination, dest_used);
+		return FALSE;
+	}
+
+	str_append_c(destination, '\t');
+	buffer_set_used_size(buf, 0);
+	bool res = dcrypt_openssl_public_key_id(pubkey, "sha256", buf, error_r);
+	dcrypt_openssl_free_public_key(&pubkey);
+	binary_to_hex_append(destination, buf->data, buf->used);
+
+	if (!res) {
+		/* well, that didn't end well */
+		buffer_set_used_size(destination, dest_used);
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static
+bool dcrypt_openssl_store_public_key_dovecot(struct dcrypt_public_key *key, buffer_t *destination, const char **error_r)
+{
+	EVP_PKEY *pubkey = (EVP_PKEY*)key;
+	unsigned char *tmp = NULL;
+
+	int rv = i2d_PUBKEY(pubkey, &tmp);
+
+	if (tmp == NULL)
+		return dcrypt_openssl_error(error_r);
+	/* then store it */
+	str_append_c(destination, '2');
+	str_append_c(destination, '\t');
+	binary_to_hex_append(destination, tmp, rv);
+	OPENSSL_free(tmp);
+
+	return TRUE;
+}
+
+static
+bool dcrypt_openssl_load_private_key(struct dcrypt_private_key **key_r, enum dcrypt_key_format format,
+	const char *data, const char *password, struct dcrypt_private_key *dec_key,
+	const char **error_r)
+{
+	EVP_PKEY *key = NULL, *key2;
+	if (format == DCRYPT_FORMAT_DOVECOT)
+		return dcrypt_openssl_load_private_key_dovecot(key_r, data, password, dec_key, error_r);
+
+	BIO *key_in = BIO_new_mem_buf((void*)data, strlen(data));
+
+	key = EVP_PKEY_new();
+
+	key2 = PEM_read_bio_PrivateKey(key_in, &key, NULL, (void*)password);
+
+	BIO_vfree(key_in);
+
+	if (key2 == NULL) {
+		EVP_PKEY_free(key);
+		return dcrypt_openssl_error(error_r);
+	}
+
+	if (EVP_PKEY_base_id(key) == EVP_PKEY_EC) {
+		EC_KEY_set_conv_form(key->pkey.ec, POINT_CONVERSION_COMPRESSED);
+	}
+
+	*key_r = (struct dcrypt_private_key *)key;
+
+	return TRUE;
+}
+
+static
+bool dcrypt_openssl_load_public_key(struct dcrypt_public_key **key_r, enum dcrypt_key_format format,
+	const char *data, const char **error_r)
+{
+	EVP_PKEY *key = NULL;
+	if (format == DCRYPT_FORMAT_DOVECOT)
+		return dcrypt_openssl_load_public_key_dovecot(key_r, data, error_r);
+
+	BIO *key_in = BIO_new_mem_buf((void*)data, strlen(data));
+
+	key = PEM_read_bio_PUBKEY(key_in, &key, NULL, NULL);
+	BIO_reset(key_in);
+	if (key == NULL) { /* ec keys are bother */
+		/* read the header */
+		char buf[27]; /* begin public key */
+		if (BIO_gets(key_in, buf, sizeof(buf)) != 1) {
+			BIO_vfree(key_in);
+			return dcrypt_openssl_error(error_r);
+		}
+		if (strcmp(buf, "-----BEGIN PUBLIC KEY-----") != 0) {
+			if (error_r != NULL)
+				*error_r = "Missing public key header";
+			return FALSE;
+		}
+		BIO *b64 = BIO_new(BIO_f_base64());
+		EC_KEY *eckey = d2i_EC_PUBKEY_bio(b64, NULL);
+		if (eckey != NULL) {
+			EC_KEY_set_conv_form(eckey, POINT_CONVERSION_COMPRESSED);
+			EC_KEY_set_asn1_flag(eckey, OPENSSL_EC_NAMED_CURVE);
+			key = EVP_PKEY_new();
+			EVP_PKEY_set1_EC_KEY(key, eckey);
+		}
+	}
+
+	BIO_vfree(key_in);
+
+	if (key == NULL)
+		return dcrypt_openssl_error(error_r);
+
+	*key_r = (struct dcrypt_public_key *)key;
+
+	return TRUE;
+}
+
+static
+bool dcrypt_openssl_store_private_key(struct dcrypt_private_key *key, enum dcrypt_key_format format,
+	const char *cipher, buffer_t *destination, const char *password, struct dcrypt_public_key *enc_key,
+	const char **error_r)
+{
+	int ec;
+	if (format == DCRYPT_FORMAT_DOVECOT) {
+		bool ret;
+		T_BEGIN {
+			ret = dcrypt_openssl_store_private_key_dovecot(key, cipher, destination, password, enc_key, error_r);
+		} T_END;
+		return ret;
+	}
+
+	EVP_PKEY *pkey = (EVP_PKEY*)key;
+	BIO *key_out = BIO_new(BIO_s_mem());
+	const EVP_CIPHER *algo = NULL;
+	if (cipher != NULL) {
+		algo = EVP_get_cipherbyname(cipher);
+		if (algo == NULL) {
+			*error_r = t_strdup_printf("Invalid cipher %s", cipher);
+			return FALSE;
+		}
+	}
+
+	ec = PEM_write_bio_PrivateKey(key_out, pkey, algo, NULL, 0, NULL, (void*)password);
+
+	BIO_flush(key_out);
+
+	if (ec != 1) {
+		BIO_vfree(key_out);
+		return dcrypt_openssl_error(error_r);
+	}
+
+	long bs;
+	char *buf;
+	bs = BIO_get_mem_data(key_out, &buf);
+	buffer_append(destination, buf, bs);
+	BIO_vfree(key_out);
+
+	return TRUE;
+}
+
+static
+bool dcrypt_openssl_store_public_key(struct dcrypt_public_key *key, enum dcrypt_key_format format, buffer_t *destination, const char **error_r)
+{
+	int ec;
+	if (format == DCRYPT_FORMAT_DOVECOT)
+		return dcrypt_openssl_store_public_key_dovecot(key, destination, error_r);
+
+	EVP_PKEY *pkey = (EVP_PKEY*)key;
+	BIO *key_out = BIO_new(BIO_s_mem());
+
+	if (EVP_PKEY_base_id(pkey) == EVP_PKEY_RSA)
+		ec = PEM_write_bio_PUBKEY(key_out, pkey);
+	else {
+		BIO *b64 = BIO_new(BIO_f_base64());
+		BIO_puts(key_out, "-----BEGIN PUBLIC KEY-----\n");
+		BIO_push(b64, key_out);
+		ec = i2d_EC_PUBKEY_bio(b64, pkey->pkey.ec);
+		BIO_flush(b64);
+		BIO_pop(b64);
+		BIO_vfree(b64);
+		BIO_puts(key_out, "-----END PUBLIC KEY-----");
+	}
+
+	if (ec != 1) {
+		BIO_vfree(key_out);
+		return dcrypt_openssl_error(error_r);
+	}
+
+	long bs;
+	char *buf;
+	bs = BIO_get_mem_data(key_out, &buf);
+	buffer_append(destination, buf, bs);
+	BIO_vfree(key_out);
+
+	return TRUE;
+}
+
+static
+bool dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r, const char **error_r)
+{
+	EVP_PKEY *pkey = (EVP_PKEY*)priv_key;
+	EVP_PKEY *pk;
+
+	if (*pub_key_r == NULL)
+		pk = EVP_PKEY_new();
+	else
+		pk = (EVP_PKEY*)*pub_key_r;
+
+	if (EVP_PKEY_base_id(pkey) == EVP_PKEY_RSA)
+	{
+		EVP_PKEY_set1_RSA(pk, RSAPublicKey_dup(pkey->pkey.rsa));
+	} else if (EVP_PKEY_base_id(pkey) == EVP_PKEY_EC) {
+		EC_KEY* eck = EVP_PKEY_get1_EC_KEY(pkey);
+		EC_KEY_set_asn1_flag(eck, OPENSSL_EC_NAMED_CURVE);
+		EVP_PKEY_set1_EC_KEY(pk, eck);
+		EC_KEY_free(eck);
+	} else {
+		*error_r = "Invalid private key";
+		return FALSE;
+	}
+
+	*pub_key_r = (struct dcrypt_public_key*)pk;
+	return TRUE;
+}
+
+static
+bool dcrypt_openssl_key_string_get_info(const char *key_data, enum dcrypt_key_format *format_r, enum dcrypt_key_version *version_r,
+	enum dcrypt_key_kind *kind_r, enum dcrypt_key_encryption_type *encryption_type_r, const char **encryption_key_hash_r,
+	const char **key_hash_r, const char **error_r)
+{
+	enum dcrypt_key_format format = DCRYPT_FORMAT_PEM;
+	enum dcrypt_key_version version = DCRYPT_KEY_VERSION_NA;
+	enum dcrypt_key_encryption_type encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_NONE;
+	enum dcrypt_key_kind kind = DCRYPT_KEY_KIND_PUBLIC;
+	const char *encryption_key_hash = NULL;
+	const char *key_hash = NULL;
+
+	if (key_data == NULL) {
+		if (error_r != NULL)
+			*error_r = "NULL key passed";
+		return FALSE;
+	}
+
+	/* is it PEM key */
+	if (strstr(key_data, "----- BEGIN ") != NULL) {
+		format = DCRYPT_FORMAT_PEM;
+		version = DCRYPT_KEY_VERSION_NA;
+		if (strstr(key_data, "ENCRYPTED") != NULL) {
+			encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_PASSWORD;
+		}
+		if (strstr(key_data, "----- BEGIN PRIVATE KEY") != NULL)
+			kind = DCRYPT_KEY_KIND_PRIVATE;
+		else if (strstr(key_data, "----- BEGIN PUBLIC KEY") != NULL)
+			kind = DCRYPT_KEY_KIND_PUBLIC;
+		else {
+			if (error_r != NULL)
+				*error_r = "Unknown/invalid PEM key type";
+			return FALSE;
+		}
+	} else T_BEGIN {
+		const char **fields = t_strsplit_tab(key_data);
+		int nfields;
+		for(nfields=0;fields[nfields]!=NULL;nfields++);
+		if (nfields < 2) {
+			if (error_r != NULL)
+				*error_r = "Unknown key format";
+			return FALSE;
+		}
+
+		format = DCRYPT_FORMAT_DOVECOT;
+
+		/* field 1 - version */
+		if (strcmp(fields[0], "1") == 0) {
+			version = DCRYPT_KEY_VERSION_1;
+			if (nfields == 3) {
+				kind = DCRYPT_KEY_KIND_PUBLIC;
+			} else if (nfields == 5 && strcmp(fields[2],"0") == 0) {
+				kind = DCRYPT_KEY_KIND_PRIVATE;
+				encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_NONE;
+			} else if (nfields == 6 && strcmp(fields[2],"2") == 0) {
+				kind = DCRYPT_KEY_KIND_PRIVATE;
+				encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_PASSWORD;
+			} else if (nfields == 11 && strcmp(fields[2],"1") == 0) {
+				kind = DCRYPT_KEY_KIND_PRIVATE;
+				encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_KEY;
+				if (encryption_key_hash_r != NULL)
+					encryption_key_hash = i_strdup(fields[nfields-2]);
+			} else {
+				if (error_r != NULL)
+					*error_r = "Invalid dovecot v1 encoding";
+				return FALSE;
+			}
+		} else if (strcmp(fields[0], "2") == 0) {
+			version = DCRYPT_KEY_VERSION_1;
+			if (nfields == 2) {
+				kind = DCRYPT_KEY_KIND_PUBLIC;
+			} else if (nfields == 5 && strcmp(fields[2],"0") == 0) {
+				kind = DCRYPT_KEY_KIND_PRIVATE;
+				encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_NONE;
+			} else if (nfields == 9 && strcmp(fields[2],"2") == 0) {
+				kind = DCRYPT_KEY_KIND_PRIVATE;
+				encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_PASSWORD;
+			} else if (nfields == 11 && strcmp(fields[2],"1") == 0) {
+				kind = DCRYPT_KEY_KIND_PRIVATE;
+				encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_KEY;
+				if (encryption_key_hash_r != NULL)
+					encryption_key_hash = i_strdup(fields[nfields-2]);
+			} else {
+				if (error_r != NULL)
+					*error_r = "Invalid dovecot v2 encoding";
+				return FALSE;
+			}
+		}
+
+		/* last field is always key hash */
+		if (key_hash_r != NULL)
+			key_hash = i_strdup(fields[nfields-1]);
+	} T_END;
+
+	if (format_r != NULL) *format_r = format;
+	if (version_r != NULL) *version_r = version;
+	if (encryption_type_r != NULL) *encryption_type_r = encryption_type;
+	if (encryption_key_hash_r != NULL) {
+		*encryption_key_hash_r = t_strdup(encryption_key_hash);
+		i_free(encryption_key_hash);
+	}
+	if (kind_r != NULL) *kind_r = kind;
+	if (key_hash_r != NULL) {
+		*key_hash_r = t_strdup(key_hash);
+		i_free(key_hash);
+	}
+	return TRUE;
+}
+
+static
+void dcrypt_openssl_free_public_key(struct dcrypt_public_key **key)
+{
+	EVP_PKEY_free((EVP_PKEY*)*key);
+	*key = NULL;
+}
+static
+void dcrypt_openssl_free_private_key(struct dcrypt_private_key **key)
+{
+	EVP_PKEY_free((EVP_PKEY*)*key);
+	*key = NULL;
+}
+static
+void dcrypt_openssl_free_keypair(struct dcrypt_keypair *keypair)
+{
+	dcrypt_openssl_free_public_key(&(keypair->pub));
+	dcrypt_openssl_free_private_key(&(keypair->priv));
+}
+
+static
+bool dcrypt_openssl_rsa_encrypt(struct dcrypt_public_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r)
+{
+	int ec;
+
+	EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new((EVP_PKEY*)key, NULL);
+	size_t outl = EVP_PKEY_size((EVP_PKEY*)key);
+	unsigned char buf[outl];
+
+	if (ctx == NULL ||
+	    EVP_PKEY_encrypt_init(ctx) < 1 ||
+	    EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) < 1 ||
+	    EVP_PKEY_encrypt(ctx, buf, &outl, data, data_len) < 1) {
+		dcrypt_openssl_error(error_r);
+		ec = -1;
+	} else {
+		buffer_append(result, buf, outl);
+		ec = 0;
+	}
+
+	EVP_PKEY_CTX_free(ctx);
+
+	return ec == 0;
+}
+static
+bool dcrypt_openssl_rsa_decrypt(struct dcrypt_private_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r)
+{
+	int ec;
+
+	EVP_PKEY_CTX *ctx = EVP_PKEY_CTX_new((EVP_PKEY*)key, NULL);
+	size_t outl = EVP_PKEY_size((EVP_PKEY*)key);
+	unsigned char buf[outl];
+
+	if (ctx == NULL ||
+	    EVP_PKEY_decrypt_init(ctx) < 1 ||
+	    EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING) < 1 ||
+	    EVP_PKEY_decrypt(ctx, buf, &outl, data, data_len) < 1) {
+		dcrypt_openssl_error(error_r);
+		ec = -1;
+	} else {
+		buffer_append(result, buf, outl);
+		ec = 0;
+	}
+
+	EVP_PKEY_CTX_free(ctx);
+
+	return ec == 0;
+}
+
+static
+const char *dcrypt_openssl_oid2name(const unsigned char *oid, size_t oid_len, const char **error_r)
+{
+	const char *name;
+	ASN1_OBJECT *obj = d2i_ASN1_OBJECT(NULL, &oid, oid_len);
+	if (obj == NULL) {
+		dcrypt_openssl_error(error_r);
+		return NULL;
+	}
+	name = OBJ_nid2sn(OBJ_obj2nid(obj));
+	ASN1_OBJECT_free(obj);
+	return name;
+}
+
+static
+bool dcrypt_openssl_name2oid(const char *name, buffer_t *oid, const char **error_r)
+{
+	ASN1_OBJECT *obj = OBJ_txt2obj(name, 0);
+	if (obj == NULL)
+		return dcrypt_openssl_error(error_r);
+	if (obj->length == 0) {
+		if (error_r != NULL)
+			*error_r = "Object has no OID assigned";
+		return FALSE;
+	}
+	unsigned char *bufptr = buffer_append_space_unsafe(oid, obj->length + 2);
+	i2d_ASN1_OBJECT(obj, &bufptr);
+	ASN1_OBJECT_free(obj);
+	if (bufptr != NULL) {
+		return TRUE;
+	}
+	return dcrypt_openssl_error(error_r);
+}
+
+static
+bool dcrypt_openssl_private_key_type(struct dcrypt_private_key *key, enum dcrypt_key_type *key_type)
+{
+	EVP_PKEY *priv = (EVP_PKEY*)key;
+	if (priv == NULL) return FALSE;
+	if (EVP_PKEY_base_id(priv) == EVP_PKEY_RSA) *key_type = DCRYPT_KEY_RSA;
+	else if (EVP_PKEY_base_id(priv) == EVP_PKEY_EC) *key_type = DCRYPT_KEY_EC;
+	return FALSE;
+}
+
+static
+bool dcrypt_openssl_public_key_type(struct dcrypt_public_key *key, enum dcrypt_key_type *key_type)
+{
+	EVP_PKEY *pub = (EVP_PKEY*)key;
+	if (pub == NULL) return FALSE;
+	if (EVP_PKEY_base_id(pub) == EVP_PKEY_RSA) *key_type = DCRYPT_KEY_RSA;
+	else if (EVP_PKEY_base_id(pub) == EVP_PKEY_EC) *key_type = DCRYPT_KEY_EC;
+	return FALSE;
+}
+
+/** this is the v1 old legacy way of doing key id's **/
+static
+bool dcrypt_openssl_public_key_id_old(struct dcrypt_public_key *key, buffer_t *result, const char **error_r)
+{
+	unsigned char buf[SHA256_DIGEST_LENGTH];
+	EVP_PKEY *pub = (EVP_PKEY*)key;
+
+	if (pub == NULL) {
+		if (error_r != NULL)
+			*error_r = "key is NULL";
+		return FALSE;
+	}
+	if (EVP_PKEY_base_id(pub) != EVP_PKEY_EC) {
+		if (error_r != NULL)
+			*error_r = "Only EC key supported";
+		return FALSE;
+	}
+
+	char *pub_pt_hex = ec_key_get_pub_point_hex(pub->pkey.ec);
+	/* digest this */
+	SHA256((const unsigned char*)pub_pt_hex, strlen(pub_pt_hex), buf);
+	buffer_append(result, buf, SHA256_DIGEST_LENGTH);
+	OPENSSL_free(pub_pt_hex);
+	return TRUE;
+}
+
+/** this is the new which uses H(der formatted public key) **/
+static
+bool dcrypt_openssl_public_key_id(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r)
+{
+	const EVP_MD *md = EVP_get_digestbyname(algorithm);
+	if (md == NULL) {
+		if (error_r != NULL)
+			*error_r = t_strdup_printf("Unknown cipher %s", algorithm);
+		return FALSE;
+	}
+	unsigned char buf[EVP_MD_size(md)];
+	EVP_PKEY *pub = (EVP_PKEY*)key;
+	const char *ptr;
+	int ec;
+	if (pub == NULL) {
+		if (error_r != NULL)
+			*error_r = "key is NULL";
+		return FALSE;
+	}
+	if (EVP_PKEY_base_id(pub) == EVP_PKEY_EC) {
+		EC_KEY_set_conv_form(pub->pkey.ec, POINT_CONVERSION_COMPRESSED);
+	}
+	BIO *b = BIO_new(BIO_s_mem());
+	if (i2d_PUBKEY_bio(b, pub) < 1) {
+		BIO_vfree(b);
+		return dcrypt_openssl_error(error_r);
+	}
+	long len = BIO_get_mem_data(b, &ptr);
+	unsigned int hlen = sizeof(buf);
+	/* then hash it */
+#if SSLEAY_VERSION_NUMBER >= 0x1010000fL
+	EVP_MD_CTX *ctx = EVP_MD_CTX_new();
+#else
+	EVP_MD_CTX *ctx = EVP_MD_CTX_create();
+#endif
+	if (EVP_DigestInit_ex(ctx, md, NULL) < 1 ||
+	    EVP_DigestUpdate(ctx, (const unsigned char*)ptr, len) < 1 ||
+	    EVP_DigestFinal_ex(ctx, buf, &hlen) < 1) {
+		ec = dcrypt_openssl_error(error_r);
+	} else {
+		buffer_append(result, buf, hlen);
+		ec = 0;
+	}
+
+#if SSLEAY_VERSION_NUMBER >= 0x1010000fL
+	EVP_MD_CTX_free(ctx);
+#else
+	EVP_MD_CTX_destroy(ctx);
+#endif
+	BIO_vfree(b);
+
+	return ec == 0;
+}
+
+static struct dcrypt_vfs dcrypt_openssl_vfs = {
+	.ctx_sym_create = dcrypt_openssl_ctx_sym_create,
+	.ctx_sym_destroy = dcrypt_openssl_ctx_sym_destroy,
+	.ctx_sym_set_key = dcrypt_openssl_ctx_sym_set_key,
+	.ctx_sym_set_iv = dcrypt_openssl_ctx_sym_set_iv,
+	.ctx_sym_set_key_iv_random = dcrypt_openssl_ctx_sym_set_key_iv_random,
+	.ctx_sym_set_padding = dcrypt_openssl_ctx_sym_set_padding,
+	.ctx_sym_get_key = dcrypt_openssl_ctx_sym_get_key,
+	.ctx_sym_get_iv = dcrypt_openssl_ctx_sym_get_iv,
+	.ctx_sym_set_aad = dcrypt_openssl_ctx_sym_set_aad,
+	.ctx_sym_get_aad = dcrypt_openssl_ctx_sym_get_aad,
+	.ctx_sym_set_tag = dcrypt_openssl_ctx_sym_set_tag,
+	.ctx_sym_get_tag = dcrypt_openssl_ctx_sym_get_tag,
+	.ctx_sym_get_key_length = dcrypt_openssl_ctx_sym_get_key_length,
+	.ctx_sym_get_iv_length = dcrypt_openssl_ctx_sym_get_iv_length,
+	.ctx_sym_get_block_size = dcrypt_openssl_ctx_sym_get_block_size,
+	.ctx_sym_init = dcrypt_openssl_ctx_sym_init,
+	.ctx_sym_update = dcrypt_openssl_ctx_sym_update,
+	.ctx_sym_final = dcrypt_openssl_ctx_sym_final,
+	.ctx_hmac_create = dcrypt_openssl_ctx_hmac_create,
+	.ctx_hmac_destroy = dcrypt_openssl_ctx_hmac_destroy,
+	.ctx_hmac_set_key = dcrypt_openssl_ctx_hmac_set_key,
+	.ctx_hmac_set_key_random = dcrypt_openssl_ctx_hmac_set_key_random,
+	.ctx_hmac_get_digest_length = dcrypt_openssl_ctx_hmac_get_digest_length,
+	.ctx_hmac_get_key = dcrypt_openssl_ctx_hmac_get_key,
+	.ctx_hmac_init = dcrypt_openssl_ctx_hmac_init,
+	.ctx_hmac_update = dcrypt_openssl_ctx_hmac_update,
+	.ctx_hmac_final = dcrypt_openssl_ctx_hmac_final,
+	.ecdh_derive_secret_local = dcrypt_openssl_ecdh_derive_secret_local,
+	.ecdh_derive_secret_peer = dcrypt_openssl_ecdh_derive_secret_peer,
+	.pbkdf2 = dcrypt_openssl_pbkdf2,
+	.generate_keypair = dcrypt_openssl_generate_keypair,
+	.load_private_key = dcrypt_openssl_load_private_key,
+	.load_public_key = dcrypt_openssl_load_public_key,
+	.store_private_key = dcrypt_openssl_store_private_key,
+	.store_public_key = dcrypt_openssl_store_public_key,
+	.private_to_public_key = dcrypt_openssl_private_to_public_key,
+	.key_string_get_info = dcrypt_openssl_key_string_get_info,
+	.free_keypair = dcrypt_openssl_free_keypair,
+	.free_public_key = dcrypt_openssl_free_public_key,
+	.free_private_key = dcrypt_openssl_free_private_key,
+	.rsa_encrypt = dcrypt_openssl_rsa_encrypt,
+	.rsa_decrypt = dcrypt_openssl_rsa_decrypt,
+	.oid2name = dcrypt_openssl_oid2name,
+	.name2oid = dcrypt_openssl_name2oid,
+	.private_key_type = dcrypt_openssl_private_key_type,
+	.public_key_type = dcrypt_openssl_public_key_type,
+	.public_key_id = dcrypt_openssl_public_key_id,
+	.public_key_id_old = dcrypt_openssl_public_key_id_old,
+};
+
+void dcrypt_openssl_init(struct module *module ATTR_UNUSED)
+{
+	OpenSSL_add_all_algorithms();
+	ERR_load_crypto_strings();
+	dcrypt_set_vfs(&dcrypt_openssl_vfs);
+}
+
+void dcrypt_openssl_deinit(void)
+{
+#if OPENSSL_API_COMPAT < 0x10100000L
+	OBJ_cleanup();
+#endif
+}
diff --git a/src/lib-dcrypt/dcrypt-private.h b/src/lib-dcrypt/dcrypt-private.h
new file mode 100644
index 00000000000..4936eb65f99
--- /dev/null
+++ b/src/lib-dcrypt/dcrypt-private.h
@@ -0,0 +1,104 @@
+#ifndef DCRYPT_PRIVATE_H
+#define DCRYPT_PRIVATE_H
+
+#define DCRYPT_DOVECOT_KEY_ENCRYPT_HASH "sha256"
+#define DCRYPT_DOVECOT_KEY_ENCRYPT_ROUNDS 2048
+
+#define DCRYPT_DOVECOT_KEY_ENCRYPT_NONE 0
+#define DCRYPT_DOVECOT_KEY_ENCRYPT_PK 1
+#define DCRYPT_DOVECOT_KEY_ENCRYPT_PASSWORD 2
+
+struct dcrypt_vfs {
+	bool (*ctx_sym_create)(const char *algorithm,
+		enum dcrypt_sym_mode mode,
+		struct dcrypt_context_symmetric **ctx_r, const char **error_r);
+	void (*ctx_sym_destroy)(struct dcrypt_context_symmetric **ctx);
+
+	void (*ctx_sym_set_key)(struct dcrypt_context_symmetric *ctx, const unsigned char *key, size_t key_len);
+	void (*ctx_sym_set_iv)(struct dcrypt_context_symmetric *ctx, const unsigned char *iv, size_t iv_len);
+	void (*ctx_sym_set_key_iv_random)(struct dcrypt_context_symmetric *ctx);
+
+	void (*ctx_sym_set_padding)(struct dcrypt_context_symmetric *ctx, bool padding);
+
+	bool (*ctx_sym_get_key)(struct dcrypt_context_symmetric *ctx, buffer_t *key);
+	bool (*ctx_sym_get_iv)(struct dcrypt_context_symmetric *ctx, buffer_t *iv);
+
+	void (*ctx_sym_set_aad)(struct dcrypt_context_symmetric *ctx, const unsigned char *aad, size_t aad_len);
+	bool (*ctx_sym_get_aad)(struct dcrypt_context_symmetric *ctx, buffer_t *aad);
+	void (*ctx_sym_set_tag)(struct dcrypt_context_symmetric *ctx, const unsigned char *tag, size_t tag_len);
+	bool (*ctx_sym_get_tag)(struct dcrypt_context_symmetric *ctx, buffer_t *tag);
+
+	unsigned int (*ctx_sym_get_key_length)(struct dcrypt_context_symmetric *ctx);
+	unsigned int (*ctx_sym_get_iv_length)(struct dcrypt_context_symmetric *ctx);
+	unsigned int (*ctx_sym_get_block_size)(struct dcrypt_context_symmetric *ctx);
+
+	bool (*ctx_sym_init)(struct dcrypt_context_symmetric *ctx, const char **error_r);
+	bool (*ctx_sym_update)(struct dcrypt_context_symmetric *ctx, const unsigned char *data, size_t data_len,
+		buffer_t *result, const char **error_r);
+	bool (*ctx_sym_final)(struct dcrypt_context_symmetric *ctx, buffer_t *result, const char **error_r);
+
+	bool (*ctx_hmac_create)(const char *algorithm, struct dcrypt_context_hmac **ctx_r, const char **error_r);
+	void (*ctx_hmac_destroy)(struct dcrypt_context_hmac **ctx);
+
+	void (*ctx_hmac_set_key)(struct dcrypt_context_hmac *ctx, const unsigned char *key, size_t key_len);
+	bool (*ctx_hmac_get_key)(struct dcrypt_context_hmac *ctx, buffer_t *key);
+	unsigned int (*ctx_hmac_get_digest_length)(struct dcrypt_context_hmac *ctx);
+	void (*ctx_hmac_set_key_random)(struct dcrypt_context_hmac *ctx);
+
+	bool (*ctx_hmac_init)(struct dcrypt_context_hmac *ctx, const char **error_r);
+	bool (*ctx_hmac_update)(struct dcrypt_context_hmac *ctx, const unsigned char *data, size_t data_len, const char **error_r);
+	bool (*ctx_hmac_final)(struct dcrypt_context_hmac *ctx, buffer_t *result, const char **error_r);
+
+	bool (*ecdh_derive_secret_local)(struct dcrypt_private_key *local_key,
+		buffer_t *R, buffer_t *S, const char **error_r);
+	bool (*ecdh_derive_secret_peer)(struct dcrypt_public_key *peer_key,
+		buffer_t *R, buffer_t *S, const char **error_r);
+	bool (*pbkdf2)(const unsigned char *password, size_t password_len,
+		const unsigned char *salt, size_t salt_len, const char *hash,
+		unsigned int rounds, buffer_t *result, unsigned int result_len,
+		const char **error_r);
+
+	bool (*generate_keypair)(struct dcrypt_keypair *pair_r, enum dcrypt_key_type kind,
+		unsigned int bits, const char *curve, const char **error_r);
+
+	bool (*load_private_key)(struct dcrypt_private_key **key_r, enum dcrypt_key_format format, const char *data,
+		const char *password, struct dcrypt_private_key *dec_key, const char **error_r);
+	bool (*load_public_key)(struct dcrypt_public_key **key_r, enum dcrypt_key_format format, const char *data,
+		const char **error_r);
+
+	bool (*store_private_key)(struct dcrypt_private_key *key, enum dcrypt_key_format format, const char *cipher, buffer_t *destination,
+		const char *password, struct dcrypt_public_key *enc_key, const char **error_r);
+	bool (*store_public_key)(struct dcrypt_public_key *key, enum dcrypt_key_format format, buffer_t *destination, const char **error_r);
+
+	bool (*private_to_public_key)(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r, const char **error_r);
+
+	bool (*key_string_get_info)(const char *key_data, enum dcrypt_key_format *format_r, enum dcrypt_key_version *version_r,
+		enum dcrypt_key_kind *kind_r, enum dcrypt_key_encryption_type *encryption_type_r, const char **encryption_key_hash_r,
+		const char **key_hash_r, const char **error_r);
+
+	void (*free_keypair)(struct dcrypt_keypair *keypair);
+	void (*free_public_key)(struct dcrypt_public_key **key);
+	void (*free_private_key)(struct dcrypt_private_key **key);
+
+	bool (*rsa_encrypt)(struct dcrypt_public_key *key, const unsigned char *data, size_t data_len,
+		buffer_t *result, const char **error_r);
+	bool (*rsa_decrypt)(struct dcrypt_private_key *key, const unsigned char *data, size_t data_len,
+		buffer_t *result, const char **error_r);
+
+	const char *(*oid2name)(const unsigned char *oid, size_t oid_len, const char **error_r);
+	bool (*name2oid)(const char *name, buffer_t *oid, const char **error_r);
+
+	bool (*private_key_type)(struct dcrypt_private_key *key, enum dcrypt_key_type *key_type);
+	bool (*public_key_type)(struct dcrypt_public_key *key, enum dcrypt_key_type *key_type);
+	bool (*public_key_id)(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r);
+	bool (*public_key_id_old)(struct dcrypt_public_key *key, buffer_t *result, const char **error_r);
+};
+
+void dcrypt_set_vfs(struct dcrypt_vfs *vfs);
+
+void dcrypt_openssl_init(struct module *module ATTR_UNUSED);
+void dcrypt_gnutls_init(struct module *module ATTR_UNUSED);
+void dcrypt_openssl_deinit(void);
+void dcrypt_gnutls_deinit(void);
+
+#endif
diff --git a/src/lib-dcrypt/dcrypt.c b/src/lib-dcrypt/dcrypt.c
new file mode 100644
index 00000000000..d4841993bbd
--- /dev/null
+++ b/src/lib-dcrypt/dcrypt.c
@@ -0,0 +1,275 @@
+#include "lib.h"
+#include "module-dir.h"
+#include "dcrypt.h"
+#include "dcrypt-private.h"
+
+static struct module *dcrypt_module = NULL;
+static struct dcrypt_vfs *dcrypt_vfs = NULL;
+
+bool dcrypt_initialize(const char *backend, const char **error_r)
+{
+	struct module_dir_load_settings mod_set;
+
+	if (dcrypt_vfs != NULL) return TRUE;
+	if (backend == NULL) backend = "openssl"; /* default for now */
+
+	const char *implementation = t_strconcat("dcrypt_",backend,NULL);
+
+	memset(&mod_set, 0, sizeof(mod_set));
+	mod_set.abi_version = DOVECOT_ABI_VERSION;
+	mod_set.require_init_funcs = 1;
+	dcrypt_module = module_dir_load(DCRYPT_MODULE_DIR, implementation, &mod_set);
+	if (dcrypt_module == NULL) {
+		if (*error_r != NULL)
+			*error_r = "No such module";
+		return FALSE;
+	}
+	if (dcrypt_module->init == NULL) {
+		if (*error_r != NULL)
+			*error_r = "Module missing init/deinit";
+		return FALSE;
+	}
+	dcrypt_module->init(dcrypt_module);
+	i_assert(dcrypt_vfs != NULL);
+	/* Destroy SSL module after(most of) the others. Especially lib-fs
+	   backends may still want to access SSL module in their own
+	   atexit-callbacks. */
+//	lib_atexit_priority(dcrypt_deinitialize, LIB_ATEXIT_PRIORITY_LOW);
+	return TRUE;
+}
+
+void dcrypt_deinitialize(void)
+{
+	i_error("I got called");
+	if (dcrypt_module != NULL) {
+		dcrypt_module->deinit();
+		module_dir_unload(&dcrypt_module);
+	}
+}
+
+void dcrypt_set_vfs(struct dcrypt_vfs *vfs)
+{
+	dcrypt_vfs = vfs;
+}
+
+bool dcrypt_ctx_sym_create(const char *algorithm, enum dcrypt_sym_mode mode, struct dcrypt_context_symmetric **ctx_r, const char **error_r)
+{
+	return dcrypt_vfs->ctx_sym_create(algorithm, mode, ctx_r, error_r);
+}
+void dcrypt_ctx_sym_destroy(struct dcrypt_context_symmetric **ctx)
+{
+	dcrypt_vfs->ctx_sym_destroy(ctx);
+}
+
+void dcrypt_ctx_sym_set_key(struct dcrypt_context_symmetric *ctx, const unsigned char *key, size_t key_len)
+{
+	dcrypt_vfs->ctx_sym_set_key(ctx, key, key_len);
+}
+void dcrypt_ctx_sym_set_iv(struct dcrypt_context_symmetric *ctx, const unsigned char *iv, size_t iv_len)
+{
+	dcrypt_vfs->ctx_sym_set_iv(ctx, iv, iv_len);
+}
+void dcrypt_ctx_sym_set_key_iv_random(struct dcrypt_context_symmetric *ctx)
+{
+	dcrypt_vfs->ctx_sym_set_key_iv_random(ctx);
+}
+
+bool dcrypt_ctx_sym_get_key(struct dcrypt_context_symmetric *ctx, buffer_t *key)
+{
+	return dcrypt_vfs->ctx_sym_get_key(ctx, key);
+}
+bool dcrypt_ctx_sym_get_iv(struct dcrypt_context_symmetric *ctx, buffer_t *iv)
+{
+	return dcrypt_vfs->ctx_sym_get_iv(ctx, iv);
+}
+
+unsigned int dcrypt_ctx_sym_get_key_length(struct dcrypt_context_symmetric *ctx)
+{
+	return dcrypt_vfs->ctx_sym_get_key_length(ctx);
+}
+
+unsigned int dcrypt_ctx_sym_get_iv_length(struct dcrypt_context_symmetric *ctx)
+{
+	return dcrypt_vfs->ctx_sym_get_iv_length(ctx);
+}
+
+void dcrypt_ctx_sym_set_aad(struct dcrypt_context_symmetric *ctx, const unsigned char *aad, size_t aad_len)
+{
+	dcrypt_vfs->ctx_sym_set_aad(ctx, aad, aad_len);
+}
+
+bool dcrypt_ctx_sym_get_aad(struct dcrypt_context_symmetric *ctx, buffer_t *aad)
+{
+	return dcrypt_vfs->ctx_sym_get_aad(ctx, aad);
+}
+
+void dcrypt_ctx_sym_set_tag(struct dcrypt_context_symmetric *ctx, const unsigned char *tag, size_t tag_len)
+{
+	dcrypt_vfs->ctx_sym_set_tag(ctx, tag, tag_len);
+}
+
+bool dcrypt_ctx_sym_get_tag(struct dcrypt_context_symmetric *ctx, buffer_t *tag)
+{
+	return dcrypt_vfs->ctx_sym_get_tag(ctx, tag);
+}
+
+unsigned int dcrypt_ctx_sym_get_block_size(struct dcrypt_context_symmetric *ctx) {
+	return dcrypt_vfs->ctx_sym_get_block_size(ctx);
+}
+
+bool dcrypt_ctx_sym_init(struct dcrypt_context_symmetric *ctx, const char **error_r)
+{
+	return dcrypt_vfs->ctx_sym_init(ctx, error_r);
+}
+bool dcrypt_ctx_sym_update(struct dcrypt_context_symmetric *ctx, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r)
+{
+	return dcrypt_vfs->ctx_sym_update(ctx, data, data_len, result, error_r);
+}
+bool dcrypt_ctx_sym_final(struct dcrypt_context_symmetric *ctx, buffer_t *result, const char **error_r)
+{
+	return dcrypt_vfs->ctx_sym_final(ctx, result, error_r);
+}
+
+void dcrypt_ctx_sym_set_padding(struct dcrypt_context_symmetric *ctx, bool padding)
+{
+	return dcrypt_vfs->ctx_sym_set_padding(ctx, padding);
+}
+
+bool dcrypt_ctx_hmac_create(const char *algorithm, struct dcrypt_context_hmac **ctx_r, const char **error_r)
+{
+	return dcrypt_vfs->ctx_hmac_create(algorithm, ctx_r, error_r);
+}
+void dcrypt_ctx_hmac_destroy(struct dcrypt_context_hmac **ctx)
+{
+	dcrypt_vfs->ctx_hmac_destroy(ctx);
+}
+
+void dcrypt_ctx_hmac_set_key(struct dcrypt_context_hmac *ctx, const unsigned char *key, size_t key_len)
+{
+	dcrypt_vfs->ctx_hmac_set_key(ctx, key, key_len);
+}
+bool dcrypt_ctx_hmac_get_key(struct dcrypt_context_hmac *ctx, buffer_t *key)
+{
+	return dcrypt_vfs->ctx_hmac_get_key(ctx, key);
+}
+void dcrypt_ctx_hmac_set_key_random(struct dcrypt_context_hmac *ctx)
+{
+	dcrypt_vfs->ctx_hmac_set_key_random(ctx);
+}
+
+unsigned int dcrypt_ctx_hmac_get_digest_length(struct dcrypt_context_hmac *ctx)
+{
+	return dcrypt_vfs->ctx_hmac_get_digest_length(ctx);
+}
+
+bool dcrypt_ctx_hmac_init(struct dcrypt_context_hmac *ctx, const char **error_r)
+{
+	return dcrypt_vfs->ctx_hmac_init(ctx, error_r);
+}
+bool dcrypt_ctx_hmac_update(struct dcrypt_context_hmac *ctx, const unsigned char *data, size_t data_len, const char **error_r)
+{
+	return dcrypt_vfs->ctx_hmac_update(ctx, data, data_len, error_r);
+}
+bool dcrypt_ctx_hmac_final(struct dcrypt_context_hmac *ctx, buffer_t *result, const char **error_r)
+{
+	return dcrypt_vfs->ctx_hmac_final(ctx, result, error_r);
+}
+
+bool dcrypt_ecdh_derive_secret_local(struct dcrypt_private_key *local_key, buffer_t *R, buffer_t *S, const char **error_r)
+{
+	return dcrypt_vfs->ecdh_derive_secret_local(local_key, R, S, error_r);
+}
+bool dcrypt_ecdh_derive_secret_peer(struct dcrypt_public_key *peer_key, buffer_t *R, buffer_t *S, const char **error_r)
+{
+	return dcrypt_vfs->ecdh_derive_secret_peer(peer_key, R, S, error_r);
+}
+
+bool dcrypt_pbkdf2(const unsigned char *password, size_t password_len, const unsigned char *salt, size_t salt_len,
+	const char *hash, unsigned int rounds, buffer_t *result, unsigned int result_len, const char **error_r)
+{
+	return dcrypt_vfs->pbkdf2(password, password_len, salt, salt_len, hash, rounds, result, result_len, error_r);
+}
+
+bool dcrypt_keypair_generate(struct dcrypt_keypair *pair_r, enum dcrypt_key_type kind, unsigned int bits, const char *curve, const char **error_r)
+{
+	return dcrypt_vfs->generate_keypair(pair_r, kind, bits, curve, error_r);
+}
+
+bool dcrypt_key_load_private(struct dcrypt_private_key **key_r, enum dcrypt_key_format format, const char *data,
+	const char *password, struct dcrypt_private_key *dec_key, const char **error_r)
+{
+	return dcrypt_vfs->load_private_key(key_r, format, data, password, dec_key, error_r);
+}
+bool dcrypt_key_load_public(struct dcrypt_public_key **key_r, enum dcrypt_key_format format, const char *data, const char **error_r)
+{
+	return dcrypt_vfs->load_public_key(key_r, format, data, error_r);
+}
+bool dcrypt_key_store_private(struct dcrypt_private_key *key, enum dcrypt_key_format format, const char *cipher, buffer_t *destination,
+	const char *password, struct dcrypt_public_key *enc_key, const char **error_r)
+{
+	return dcrypt_vfs->store_private_key(key, format, cipher, destination, password, enc_key, error_r);
+}
+bool dcrypt_key_store_public(struct dcrypt_public_key *key, enum dcrypt_key_format format, buffer_t *destination, const char **error_r)
+{
+	return dcrypt_vfs->store_public_key(key, format, destination, error_r);
+}
+
+bool dcrypt_key_convert_private_to_public(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r, const char **error_r)
+{
+	return dcrypt_vfs->private_to_public_key(priv_key, pub_key_r, error_r);
+}
+bool dcrypt_key_string_get_info(const char *key_data, enum dcrypt_key_format *format_r, enum dcrypt_key_version *version_r,
+	enum dcrypt_key_kind *kind_r, enum dcrypt_key_encryption_type *encryption_type_r, const char **encryption_key_hash_r,
+	const char **key_hash_r, const char **error_r)
+{
+	return dcrypt_vfs->key_string_get_info(key_data, format_r, version_r, kind_r, encryption_type_r,
+		encryption_key_hash_r, key_hash_r, error_r);
+}
+
+bool dcrypt_key_type_private(struct dcrypt_private_key *key, enum dcrypt_key_type *type)
+{
+	return dcrypt_vfs->private_key_type(key, type);
+}
+bool dcrypt_key_type_public(struct dcrypt_public_key *key, enum dcrypt_key_type *type)
+{
+	return dcrypt_vfs->public_key_type(key, type);
+}
+bool dcrypt_key_id_public(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r)
+{
+	return dcrypt_vfs->public_key_id(key, algorithm, result, error_r);
+}
+bool dcrypt_key_id_public_old(struct dcrypt_public_key *key, buffer_t *result, const char **error_r)
+{
+	return dcrypt_vfs->public_key_id_old(key, result, error_r);
+}
+void dcrypt_keypair_free(struct dcrypt_keypair *keypair)
+{
+	dcrypt_vfs->free_keypair(keypair);
+}
+void dcrypt_key_free_public(struct dcrypt_public_key **key)
+{
+	dcrypt_vfs->free_public_key(key);
+}
+void dcrypt_key_free_private(struct dcrypt_private_key **key)
+{
+	dcrypt_vfs->free_private_key(key);
+}
+
+bool dcrypt_rsa_encrypt(struct dcrypt_public_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r)
+{
+	return dcrypt_vfs->rsa_encrypt(key, data, data_len, result, error_r);
+}
+bool dcrypt_rsa_decrypt(struct dcrypt_private_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r)
+{
+	return dcrypt_vfs->rsa_decrypt(key, data, data_len, result, error_r);
+}
+
+const char *dcrypt_oid2name(const unsigned char *oid, size_t oid_len, const char **error_r)
+{
+	return dcrypt_vfs->oid2name(oid, oid_len, error_r);
+}
+bool dcrypt_name2oid(const char *name, buffer_t *oid, const char **error_r)
+{
+	return dcrypt_vfs->name2oid(name, oid, error_r);
+}
+
diff --git a/src/lib-dcrypt/dcrypt.h b/src/lib-dcrypt/dcrypt.h
new file mode 100644
index 00000000000..07138d29a8f
--- /dev/null
+++ b/src/lib-dcrypt/dcrypt.h
@@ -0,0 +1,204 @@
+#ifndef DCRYPT_H
+#define DCRYPT_H 1
+
+struct dcrypt_context_symmetric;
+struct dcrypt_context_hmac;
+struct dcrypt_public_key;
+struct dcrypt_private_key;
+
+struct dcrypt_keypair {
+	struct dcrypt_public_key *pub;
+	struct dcrypt_private_key *priv;
+};
+
+enum dcrypt_sym_mode {
+	DCRYPT_MODE_ENCRYPT,
+	DCRYPT_MODE_DECRYPT
+};
+
+enum dcrypt_key_type {
+	DCRYPT_KEY_RSA = 0x1,
+	DCRYPT_KEY_EC  = 0x2
+};
+
+/**
+ * dovecot key format:
+ * version tab version-specific data
+ * v1: version tab nid tab raw ec private key (in hex)
+ * v2: version tab algorithm oid tab private-or-public-key-only (in hex)
+ */
+enum dcrypt_key_format {
+	DCRYPT_FORMAT_PEM,
+	DCRYPT_FORMAT_DOVECOT,
+};
+
+enum dcrypt_key_encryption_type {
+	DCRYPT_KEY_ENCRYPTION_TYPE_NONE,
+	DCRYPT_KEY_ENCRYPTION_TYPE_KEY,
+	DCRYPT_KEY_ENCRYPTION_TYPE_PASSWORD
+};
+
+enum dcrypt_key_version {
+	DCRYPT_KEY_VERSION_1,
+	DCRYPT_KEY_VERSION_2,
+	DCRYPT_KEY_VERSION_NA /* not applicable, PEM key */
+};
+
+enum dcrypt_key_kind {
+	DCRYPT_KEY_KIND_PUBLIC,
+	DCRYPT_KEY_KIND_PRIVATE
+};
+
+/**
+ * load and initialize dcrypt backend, use either openssl or gnutls
+ */
+bool dcrypt_initialize(const char *backend, const char **error_r);
+
+/**
+ * deinitialize dcrypt
+ */
+void dcrypt_deinitialize(void);
+
+/**
+ * create symmetric context
+ */
+bool dcrypt_ctx_sym_create(const char *algorithm, enum dcrypt_sym_mode mode, struct dcrypt_context_symmetric **ctx_r, const char **error_r);
+
+/**
+ * destroy symmetric context and free memory
+ */
+void dcrypt_ctx_sym_destroy(struct dcrypt_context_symmetric **ctx);
+
+/**
+ * key and IV manipulation functions
+ */
+void dcrypt_ctx_sym_set_key(struct dcrypt_context_symmetric *ctx, const unsigned char *key, size_t key_len);
+void dcrypt_ctx_sym_set_iv(struct dcrypt_context_symmetric *ctx, const unsigned char *iv, size_t iv_len);
+void dcrypt_ctx_sym_set_key_iv_random(struct dcrypt_context_symmetric *ctx);
+bool dcrypt_ctx_sym_get_key(struct dcrypt_context_symmetric *ctx, buffer_t *key);
+bool dcrypt_ctx_sym_get_iv(struct dcrypt_context_symmetric *ctx, buffer_t *iv);
+
+/**
+ * turn padding on/off (default: on)
+ */
+void dcrypt_ctx_sym_set_padding(struct dcrypt_context_symmetric *ctx, bool padding);
+
+
+/**
+ * authentication data manipulation (use with GCM only)
+ */
+void dcrypt_ctx_sym_set_aad(struct dcrypt_context_symmetric *ctx, const unsigned char *aad, size_t aad_len);
+bool dcrypt_ctx_sym_get_aad(struct dcrypt_context_symmetric *ctx, buffer_t *aad);
+/**
+ * result tag from aead (use with GCM only)
+ */
+void dcrypt_ctx_sym_set_tag(struct dcrypt_context_symmetric *ctx, const unsigned char *tag, size_t tag_len);
+bool dcrypt_ctx_sym_get_tag(struct dcrypt_context_symmetric *ctx, buffer_t *tag);
+
+/* get various lengths */
+unsigned int dcrypt_ctx_sym_get_key_length(struct dcrypt_context_symmetric *ctx);
+unsigned int dcrypt_ctx_sym_get_iv_length(struct dcrypt_context_symmetric *ctx);
+unsigned int dcrypt_ctx_sym_get_block_size(struct dcrypt_context_symmetric *ctx);
+
+/**
+ * initialize crypto
+ */
+bool dcrypt_ctx_sym_init(struct dcrypt_context_symmetric *ctx, const char **error_r);
+/**
+ * update with data
+ */
+bool dcrypt_ctx_sym_update(struct dcrypt_context_symmetric *ctx, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r);
+/**
+ * perform final step (may or may not emit data)
+ */
+bool dcrypt_ctx_sym_final(struct dcrypt_context_symmetric *ctx, buffer_t *result, const char **error_r);
+
+/**
+ * create HMAC context, algorithm is digest algorithm
+ */
+bool dcrypt_ctx_hmac_create(const char *algorithm, struct dcrypt_context_hmac **ctx_r, const char **error_r);
+/**
+ * destroy HMAC context and free memory
+ */
+void dcrypt_ctx_hmac_destroy(struct dcrypt_context_hmac **ctx);
+
+/**
+ * hmac key manipulation
+ */
+void dcrypt_ctx_hmac_set_key(struct dcrypt_context_hmac *ctx, const unsigned char *key, size_t key_len);
+bool dcrypt_ctx_hmac_get_key(struct dcrypt_context_hmac *ctx, buffer_t *key);
+void dcrypt_ctx_hmac_set_key_random(struct dcrypt_context_hmac *ctx);
+
+/**
+ * get digest length for HMAC
+ */
+unsigned int dcrypt_ctx_hmac_get_digest_length(struct dcrypt_context_hmac *ctx);
+
+/**
+ * initialize hmac
+ */
+bool dcrypt_ctx_hmac_init(struct dcrypt_context_hmac *ctx, const char **error_r);
+/**
+ * update hmac context with data
+ */
+bool dcrypt_ctx_hmac_update(struct dcrypt_context_hmac *ctx, const unsigned char *data, size_t data_len, const char **error_r);
+/**
+ * perform final rounds and retrieve result
+ */
+bool dcrypt_ctx_hmac_final(struct dcrypt_context_hmac *ctx, buffer_t *result, const char **error_r);
+
+
+/**
+ * Elliptic Curve based Diffie-Heffman shared secret derivation */
+bool dcrypt_ecdh_derive_secret_local(struct dcrypt_private_key *local_key, buffer_t *R, buffer_t *S, const char **error_r);
+bool dcrypt_ecdh_derive_secret_peer(struct dcrypt_public_key *peer_key, buffer_t *R, buffer_t *S, const char **error_r);
+
+/**
+ * generate cryptographic data from password and salt. Use 1000-10000 for rounds.
+ */
+bool dcrypt_pbkdf2(const unsigned char *password, size_t password_len, const unsigned char *salt, size_t salt_len,
+	const char *hash, unsigned int rounds, buffer_t *result, unsigned int result_len, const char **error_r);
+
+bool dcrypt_keypair_generate(struct dcrypt_keypair *pair_r, enum dcrypt_key_type kind, unsigned int bits, const char *curve, const char **error_r);
+
+/**
+ * load loads key structure from external format. 
+ * store stores key structure into external format.
+ *
+ * you can provide either PASSWORD or ENC_KEY, not both.
+ */
+bool dcrypt_key_load_private(struct dcrypt_private_key **key_r, enum dcrypt_key_format format, const char *data, 
+	const char *password, struct dcrypt_private_key *dec_key, const char **error_r);
+
+bool dcrypt_key_load_public(struct dcrypt_public_key **key_r, enum dcrypt_key_format format, const char *data, const char **error_r);
+
+bool dcrypt_key_store_private(struct dcrypt_private_key *key, enum dcrypt_key_format format, const char *cipher, 
+	buffer_t *destination, const char *password, struct dcrypt_public_key *enc_key, const char **error_r);
+
+bool dcrypt_key_store_public(struct dcrypt_public_key *key, enum dcrypt_key_format format, buffer_t *destination, const char **error_r);
+
+bool dcrypt_key_convert_private_to_public(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r, const char **error_r);
+
+void dcrypt_keypair_free(struct dcrypt_keypair *keypair);
+
+void dcrypt_key_free_public(struct dcrypt_public_key **key);
+void dcrypt_key_free_private(struct dcrypt_private_key **key);
+
+bool dcrypt_key_type_private(struct dcrypt_private_key *key, enum dcrypt_key_type *type);
+bool dcrypt_key_type_public(struct dcrypt_public_key *key, enum dcrypt_key_type *type);
+bool dcrypt_key_id_public(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r); /* return digest of key */
+bool dcrypt_key_id_public_old(struct dcrypt_public_key *key, buffer_t *result, const char **error_r); /* return SHA1 sum of key */
+
+bool dcrypt_key_string_get_info(const char *key_data, enum dcrypt_key_format *format_r, enum dcrypt_key_version *version_r,
+	enum dcrypt_key_kind *kind_r, enum dcrypt_key_encryption_type *encryption_type_r, const char **encryption_key_hash_r,
+	const char **key_hash_r, const char **error_r);
+
+/* RSA stuff */
+bool dcrypt_rsa_encrypt(struct dcrypt_public_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r);
+bool dcrypt_rsa_decrypt(struct dcrypt_private_key *key, const unsigned char *data, size_t data_len, buffer_t *result, const char **error_r);
+
+/* OID stuff */
+const char *dcrypt_oid2name(const unsigned char *oid, size_t oid_len, const char **error_r);
+bool dcrypt_name2oid(const char *name, buffer_t *oid, const char **error_r);
+
+#endif
diff --git a/src/lib-dcrypt/istream-decrypt.c b/src/lib-dcrypt/istream-decrypt.c
new file mode 100644
index 00000000000..d8c6329aaa9
--- /dev/null
+++ b/src/lib-dcrypt/istream-decrypt.c
@@ -0,0 +1,873 @@
+#include "lib.h"
+#include "buffer.h"
+#include "randgen.h"
+#include "safe-memset.h"
+#include "hash-method.h"
+#include "sha2.h"
+#include "dcrypt.h"
+#include "istream.h"
+#include "istream-decrypt.h"
+#include "istream-private.h"
+#include "dcrypt-iostream-private.h"
+
+#include "hex-binary.h"
+
+#include <arpa/inet.h>
+
+#define ISTREAM_DECRYPT_READ_FIRST 15
+
+enum io_stream_encrypt_flags {
+	IO_STREAM_ENC_INTEGRITY_HMAC = 0x1,
+	IO_STREAM_ENC_INTEGRITY_AEAD = 0x2,
+	IO_STREAM_ENC_INTEGRITY_NONE = 0x4,
+	IO_STREAM_ENC_VERSION_1      = 0x8,
+};
+
+struct decrypt_istream {
+	struct istream_private istream;
+	buffer_t *buf;
+
+	i_stream_decrypt_get_key_callback_t *key_callback;
+	void *key_context;
+
+	struct dcrypt_private_key *priv_key;
+	bool initialized;
+	bool finalized;
+	bool use_mac;
+
+	uoff_t ftr, pos;
+	enum io_stream_encrypt_flags flags;
+
+	unsigned char *iv;  /* original iv, in case seeking is done, future feature */
+
+	struct dcrypt_context_symmetric *ctx_sym;
+	struct dcrypt_context_hmac *ctx_mac;
+
+	enum {
+		DECRYPT_FORMAT_V1,
+		DECRYPT_FORMAT_V2
+	} format;
+};
+
+static
+ssize_t i_stream_decrypt_read_header_v1(struct decrypt_istream *stream,
+	const unsigned char *data, size_t mlen)
+{
+	const char *error = NULL;
+	size_t hdr_len = 0;
+	uint16_t len;
+	int ec, i = 0;
+
+	const unsigned char *digest_pos = NULL, *key_digest_pos = NULL, *key_ct_pos = NULL;
+
+	size_t pos = 7;
+	size_t digest_len = 0;
+	size_t key_ct_len = 0;
+	size_t key_digest_size = 0;
+
+	buffer_t ephemeral_key;
+	buffer_t *secret = buffer_create_dynamic(pool_datastack_create(), 256);
+	buffer_t *key = buffer_create_dynamic(pool_datastack_create(), 256);
+
+	hdr_len = ((data[0] << 8) | data[1]) + 12;
+	if (mlen < hdr_len) {
+		/* try to read more */
+		return 0;
+	}
+
+	data+=2;
+	mlen-=2;
+
+	memcpy(&len, data, 2);
+
+	while(i < 4 && mlen > 2 && (len = ntohs(len)) <= (mlen - 2) && len > 0) {
+		data += 2;
+		mlen -= 2;
+		pos += 2;
+
+		switch(i++) {
+		case 0:
+			buffer_create_from_const_data(&ephemeral_key, data, len);
+			break;
+		case 1:
+			/* public key id */
+			digest_pos = data;
+			digest_len = len;
+			break;
+		case 2:
+			/* encryption key digest */
+			key_digest_pos = data;
+			key_digest_size = len;
+			break;
+		case 3:
+			/* encrypted key data */
+			key_ct_pos = data;
+			key_ct_len = len;
+			break;
+		}
+		pos += len;
+		data += len;
+		mlen -= len;
+		memcpy(&len, data, 2);
+	}
+
+	if (i < 4) {
+		io_stream_set_error(&stream->istream.iostream, "Invalid or corrupted header");
+		return -1;
+	}
+
+	/* we don't have a private key */
+	if (stream->priv_key == NULL) {
+		/* see if we can get one */
+		if (stream->key_callback != NULL) {
+			unsigned char *key_id = t_malloc(digest_len);
+			memcpy(key_id, digest_pos, digest_len);
+			int ret = stream->key_callback(key_id, &(stream->priv_key), &error, stream->key_context);
+			if (ret < 0) {
+				io_stream_set_error(&stream->istream.iostream, "Private key not available: %s", error);
+				return -1;
+			}
+			if (ret == 0) {
+				io_stream_set_error(&stream->istream.iostream, "Private key not available");
+				return -1;
+			}
+		} else {
+			io_stream_set_error(&stream->istream.iostream, "Private key not available");
+			return -1;
+		}
+	}
+
+	buffer_t *check = buffer_create_dynamic(pool_datastack_create(), 32);
+	struct dcrypt_public_key *pubkey = NULL;
+
+	/* do we have correct private key? */
+	if (!dcrypt_key_convert_private_to_public(stream->priv_key, &pubkey, &error)) {
+		io_stream_set_error(&stream->istream.iostream, "Cannot convert private key to public: %s", error);
+		return -1;
+	}
+	ec = 0;
+	if (!dcrypt_key_id_public_old(pubkey, check, &error)) {
+		io_stream_set_error(&stream->istream.iostream, "Cannot get public key hash: %s", error);
+		ec = -1;
+	} else {
+		if (memcmp(digest_pos, check->data, I_MIN(digest_len,check->used)) != 0) {
+			io_stream_set_error(&stream->istream.iostream, "Private key not available");
+			ec = -1;
+		}
+	}
+	dcrypt_key_free_public(&pubkey);
+	if (ec != 0) return ec;
+
+	/* derive shared secret */
+	if (!dcrypt_ecdh_derive_secret_local(stream->priv_key, &ephemeral_key, secret, &error)) {
+		io_stream_set_error(&stream->istream.iostream, "Cannot perform ECDH: %s", error);
+		return -1;
+	}
+
+	/* run it thru SHA256 once */
+	const struct hash_method *hash = &hash_method_sha256;
+	unsigned char hctx[hash->context_size];
+	unsigned char hres[hash->digest_size];
+	hash->init(hctx);
+	hash->loop(hctx, secret->data, secret->used);
+	hash->result(hctx, hres);
+	safe_memset(buffer_get_modifiable_data(secret, 0), 0, secret->used);
+
+	/* NB! The old code was broken and used this kind of IV - it is not correct, but
+	   we need to stay compatible with old data */
+
+	/* use it to decrypt the actual encryption key */
+	struct dcrypt_context_symmetric *dctx;
+	if (!dcrypt_ctx_sym_create("aes-256-ctr", DCRYPT_MODE_DECRYPT, &dctx, &error)) {
+		io_stream_set_error(&stream->istream.iostream, "Key decryption error: %s", error);
+		return -1;
+	}
+
+	ec = 0;
+	dcrypt_ctx_sym_set_iv(dctx, (const unsigned char*)"\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0", 16);
+	dcrypt_ctx_sym_set_key(dctx, hres, hash->digest_size);
+	if (!dcrypt_ctx_sym_init(dctx, &error) ||
+	    !dcrypt_ctx_sym_update(dctx, key_ct_pos, key_ct_len, key, &error) ||
+	    !dcrypt_ctx_sym_final(dctx, key, &error)) {
+		io_stream_set_error(&stream->istream.iostream, "Key decryption error: %s", error);
+		ec = -1;
+	}
+	dcrypt_ctx_sym_destroy(&dctx);
+
+	if (ec != 0) {
+		io_stream_set_error(&stream->istream.iostream, "Key decryption error: %s", error);
+		return -1;
+	}
+
+	/* see if we got the correct key */
+	hash->init(hctx);
+	hash->loop(hctx, key->data, key->used);
+	hash->result(hctx, hres);
+
+	if (key_digest_size != sizeof(hres)) {
+		io_stream_set_error(&stream->istream.iostream, "Key decryption error: invalid digest length");
+		return -1;
+	}
+	if (memcmp(hres, key_digest_pos, sizeof(hres)) != 0) {
+		io_stream_set_error(&stream->istream.iostream, "Key decryption error: decrypted key is invalid");
+		return -1;
+	}
+
+	/* prime context with key */
+	if (!dcrypt_ctx_sym_create("aes-256-ctr", DCRYPT_MODE_DECRYPT, &(stream->ctx_sym), &error)) {
+		io_stream_set_error(&stream->istream.iostream, "Decryption context create error: %s", error);
+		return -1;
+	}
+
+	/* Again, old code used this IV, so we have to use it too */
+	dcrypt_ctx_sym_set_iv(stream->ctx_sym, (const unsigned char*)"\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0\x0", 16);
+	dcrypt_ctx_sym_set_key(stream->ctx_sym, key->data, key->used);
+
+	safe_memset(buffer_get_modifiable_data(key, 0), 0, key->used);
+
+	if (!dcrypt_ctx_sym_init(stream->ctx_sym, &error)) {
+		io_stream_set_error(&stream->istream.iostream, "Decryption init error: %s", error);
+		return -1;
+	}
+
+	stream->use_mac = FALSE;
+	stream->initialized = TRUE;
+	/* now we are ready to decrypt stream */
+
+	return hdr_len;
+}
+
+static bool get_msb32(const unsigned char **_data, const unsigned char *end, uint32_t *num_r)
+{
+	const unsigned char *data = *_data;
+	if (end-data < 4)
+		return FALSE;
+	*num_r = ((uint32_t)data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3];
+	*_data += 4;
+	return TRUE;
+}
+
+static
+bool i_stream_decrypt_der(const unsigned char **_data,
+			  const unsigned char *end, const char **str_r)
+{
+	const unsigned char *data = *_data;
+	unsigned int len;
+
+	if (end-data < 2)
+		return FALSE;
+	/* get us DER encoded length */
+	if ((data[1] & 0x80) != 0) {
+		/* two byte length */
+		if (end-data < 3)
+			return FALSE;
+		len = ((data[1] & 0x7f) << 8) + data[2] + 3;
+	} else {
+		len = data[1] + 2;
+	}
+	if ((size_t)(end-data) < len)
+		return FALSE;
+	*str_r = dcrypt_oid2name(data, len, NULL);
+	*_data += len;
+	return TRUE;
+}
+
+static
+ssize_t i_stream_decrypt_key(struct decrypt_istream *stream, const char *malg, unsigned int rounds,
+	const unsigned char *data, const unsigned char *end, buffer_t *key, size_t key_len)
+{
+	const char *error;
+	enum dcrypt_key_type ktype;
+	int keys;
+	bool have_key = FALSE;
+	unsigned char dgst[32];
+	uint32_t val;
+	buffer_t buf;
+
+	if (data == end)
+		return 0;
+
+	keys = *data++;
+
+	/* if we have a key, prefab the digest */
+	if (stream->key_callback == NULL) {
+		if (stream->priv_key == NULL) {	
+			io_stream_set_error(&stream->istream.iostream, "Decryption error: no private key available");
+			return -1;
+		}
+		buffer_create_from_data(&buf, dgst, sizeof(dgst));
+		struct dcrypt_public_key *pub = NULL;
+		dcrypt_key_convert_private_to_public(stream->priv_key, &pub, NULL);
+		dcrypt_key_id_public(pub, "sha256", &buf, NULL);
+		dcrypt_key_free_public(&pub);
+	}
+
+	/* for each key */
+	for(;keys>0;keys--) {
+		if ((size_t)(end-data) < 1 + (ssize_t)sizeof(dgst))
+			return 0;
+		ktype = *data++;
+
+		if (stream->key_callback != NULL) {
+			memcpy(dgst, data, sizeof(dgst));
+			/* hope you going to give us right key.. */
+			int ret = stream->key_callback(dgst, &(stream->priv_key), &error, stream->key_context);
+			if (ret < 0) {
+				io_stream_set_error(&stream->istream.iostream, "Private key not available: %s", error);
+				return -1;
+			}
+			if (ret > 0) {
+				have_key = TRUE;
+				break;
+			}
+		} else {
+			/* see if key matches to the one we have */
+			if (memcmp(dgst, data, sizeof(dgst)) == 0) {
+			      	have_key = TRUE;
+				break;
+			}
+		}
+		data += sizeof(dgst);
+
+		/* wasn't correct key, skip over some data */
+		if (!get_msb32(&data, end, &val) ||
+		    !get_msb32(&data, end, &val))
+			return 0;
+	}
+
+	/* didn't find matching key */
+	if (!have_key) {
+		io_stream_set_error(&stream->istream.iostream, "Decryption error: no private key available");
+		return -1;
+	}
+
+	data += sizeof(dgst);
+
+	const unsigned char *ephemeral_key;
+	uint32_t ep_key_len;
+	const unsigned char *encrypted_key;
+	uint32_t eklen;
+	const unsigned char *ekhash;
+	uint32_t ekhash_len;
+
+	/* read ephemeral key (can be missing for RSA) */
+	if (!get_msb32(&data, end, &ep_key_len) || (size_t)(end-data) < ep_key_len)
+		return 0;
+	ephemeral_key = data;
+	data += ep_key_len;
+
+	/* read encrypted key */
+	if (!get_msb32(&data, end, &eklen) || (size_t)(end-data) < eklen)
+		return 0;
+	encrypted_key = data;
+	data += eklen;
+
+	/* read key data hash */
+	if (!get_msb32(&data, end, &ekhash_len) || (size_t)(end-data) < ekhash_len)
+		return 0;
+	ekhash = data;
+	data += ekhash_len;
+
+	/* decrypt the seed */
+	if (ktype == DCRYPT_KEY_RSA) {
+		if (!dcrypt_rsa_decrypt(stream->priv_key, encrypted_key, eklen, key, &error)) {
+			io_stream_set_error(&stream->istream.iostream, "key decryption error: %s", error);
+			return -1;
+		}
+	} else if (ktype == DCRYPT_KEY_EC) {
+		/* perform ECDHE */
+		buffer_t *temp_key = buffer_create_dynamic(pool_datastack_create(), 256);
+		buffer_t *secret = buffer_create_dynamic(pool_datastack_create(), 256);
+		buffer_t peer_key;
+		buffer_create_from_const_data(&peer_key, ephemeral_key, ep_key_len);
+		if (!dcrypt_ecdh_derive_secret_local(stream->priv_key, &peer_key, secret, &error)) {
+			io_stream_set_error(&stream->istream.iostream, "Key decryption error: corrupted header");
+			return -1;
+		}
+
+		/* use shared secret and peer key to generate decryption key, AES-256-CBC has 32 byte key and 16 byte IV */
+		if (!dcrypt_pbkdf2(secret->data, secret->used, peer_key.data, peer_key.used,
+		    malg, rounds, temp_key, 32+16, &error)) {
+			safe_memset(buffer_get_modifiable_data(secret, 0), 0, secret->used);
+			io_stream_set_error(&stream->istream.iostream, "Key decryption error: %s", error);
+			return -1;
+		}
+
+		safe_memset(buffer_get_modifiable_data(secret, 0), 0, secret->used);
+		if (temp_key->used != 32+16) {
+			safe_memset(buffer_get_modifiable_data(temp_key, 0), 0, temp_key->used);
+			io_stream_set_error(&stream->istream.iostream, "Cannot perform key decryption: invalid temporary key");
+			return -1;
+		}
+		struct dcrypt_context_symmetric *dctx;
+		if (!dcrypt_ctx_sym_create("AES-256-CBC", DCRYPT_MODE_DECRYPT, &dctx, &error)) {
+			safe_memset(buffer_get_modifiable_data(temp_key, 0), 0, temp_key->used);
+			io_stream_set_error(&stream->istream.iostream, "Key decryption error: %s", error);
+			return -1;
+		}
+		const unsigned char *ptr = temp_key->data;
+
+		/* we use ephemeral_key for IV */
+		dcrypt_ctx_sym_set_key(dctx, ptr, 32);
+		dcrypt_ctx_sym_set_iv(dctx, ptr+32, 16);
+		safe_memset(buffer_get_modifiable_data(temp_key, 0), 0, temp_key->used);
+
+		int ec = 0;
+		if (!dcrypt_ctx_sym_init(dctx, &error) ||
+		    !dcrypt_ctx_sym_update(dctx, encrypted_key, eklen, key, &error) ||
+		    !dcrypt_ctx_sym_final(dctx, key, &error)) {
+			io_stream_set_error(&stream->istream.iostream, "Cannot perform key decryption: %s", error);
+			ec = -1;
+		}
+
+		if (key->used != key_len) {
+			io_stream_set_error(&stream->istream.iostream, "Cannot perform key decryption: invalid key length");
+			ec = -1;
+		}
+
+		dcrypt_ctx_sym_destroy(&dctx);
+		if (ec != 0) return ec;
+	} else {
+		io_stream_set_error(&stream->istream.iostream, "Decryption error: unsupported key type 0x%02x", ktype);
+		return -1;
+	}
+
+	/* make sure we were able to decrypt the encrypted key correctly */
+	const struct hash_method *hash = hash_method_lookup(t_str_lcase(malg));
+	if (hash == NULL) {
+		safe_memset(buffer_get_modifiable_data(key, 0), 0, key->used);
+		io_stream_set_error(&stream->istream.iostream, "Decryption error: unsupported hash algorithm: %s", malg);
+		return -1;
+	}
+	unsigned char hctx[hash->context_size];
+	unsigned char hres[hash->digest_size];
+	hash->init(hctx);
+	hash->loop(hctx, key->data, key->used);
+	hash->result(hctx, hres);
+
+	for(int i = 1; i < 2049; i++) {
+		uint32_t i_msb = htonl(i);
+
+		hash->init(hctx);
+		hash->loop(hctx, hres, sizeof(hres));
+		hash->loop(hctx, &i_msb, sizeof(i_msb));
+		hash->result(hctx, hres);
+	}
+
+	/* do the comparison */
+	if (memcmp(ekhash, hres, I_MIN(ekhash_len, sizeof(hres))) != 0) {
+		safe_memset(buffer_get_modifiable_data(key, 0), 0, key->used);
+		io_stream_set_error(&stream->istream.iostream, "Decryption error: corrupted header ekhash");
+		return -1;
+	}
+	return 1;
+}
+
+static
+int i_stream_decrypt_header_contents(struct decrypt_istream *stream,
+				     const unsigned char *data, size_t size)
+{
+	const unsigned char *end = data + size;
+	bool failed = FALSE;
+
+	/* read cipher OID */
+	const char *calg;
+	if (!i_stream_decrypt_der(&data, end, &calg))
+		return 0;
+	if (calg == NULL || !dcrypt_ctx_sym_create(calg, DCRYPT_MODE_DECRYPT, &(stream->ctx_sym), NULL)) {
+		io_stream_set_error(&stream->istream.iostream, "Decryption error: unsupported/invalid cipher: %s", calg);
+		return -1;
+	}
+
+	/* read MAC oid (MAC is used for PBKDF2 and key data digest, too) */
+	const char *malg;
+	if (!i_stream_decrypt_der(&data, end, &malg))
+		return 0;
+	if (malg == NULL || !dcrypt_ctx_hmac_create(malg, &(stream->ctx_mac), NULL)) {
+		io_stream_set_error(&stream->istream.iostream, "Decryption error: unsupported/invalid MAC algorithm: %s", malg);
+		return -1;
+	}
+
+	/* read rounds (for PBKDF2) */
+	uint32_t rounds;
+	if (!get_msb32(&data, end, &rounds))
+		return 0;
+	/* read key data length */
+	uint32_t kdlen;
+	if (!get_msb32(&data, end, &kdlen))
+		return 0;
+
+	size_t tagsize;
+
+	if ((stream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) {
+		tagsize = IOSTREAM_TAG_SIZE;
+	} else if ((stream->flags & IO_STREAM_ENC_INTEGRITY_AEAD) == IO_STREAM_ENC_INTEGRITY_AEAD) {
+		tagsize = IOSTREAM_TAG_SIZE;
+	} else {
+		tagsize = 0;
+	}
+
+	/* how much key data we should be getting */
+	size_t kl = dcrypt_ctx_sym_get_key_length(stream->ctx_sym) + dcrypt_ctx_sym_get_iv_length(stream->ctx_sym) + tagsize;
+	buffer_t *keydata = buffer_create_dynamic(pool_datastack_create(), kl);
+
+	/* try to decrypt the keydata with a private key */
+	int ret;
+	if ((ret = i_stream_decrypt_key(stream, malg, rounds, data, end, keydata, kl)) <= 0)
+		return ret;
+
+	/* oh, it worked! */
+	const unsigned char *ptr = keydata->data;
+	if (keydata->used != kl) {
+		/* but returned wrong amount of data */
+		io_stream_set_error(&stream->istream.iostream, "Key decryption error: Key data length mismatch");
+		return -1;
+	}
+
+	/* prime contexts */
+	dcrypt_ctx_sym_set_key(stream->ctx_sym, ptr, dcrypt_ctx_sym_get_key_length(stream->ctx_sym));
+	ptr += dcrypt_ctx_sym_get_key_length(stream->ctx_sym);
+	dcrypt_ctx_sym_set_iv(stream->ctx_sym, ptr, dcrypt_ctx_sym_get_iv_length(stream->ctx_sym));
+	stream->iv = i_malloc(dcrypt_ctx_sym_get_iv_length(stream->ctx_sym));
+	memcpy(stream->iv, ptr, dcrypt_ctx_sym_get_iv_length(stream->ctx_sym));
+	ptr += dcrypt_ctx_sym_get_iv_length(stream->ctx_sym);
+
+	/* based on the chosen MAC, initialize HMAC or AEAD */
+	if ((stream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) {
+		const char *error;
+		dcrypt_ctx_hmac_set_key(stream->ctx_mac, ptr, tagsize);
+		if (!dcrypt_ctx_hmac_init(stream->ctx_mac, &error)) {
+			io_stream_set_error(&stream->istream.iostream, "MAC error: %s", error);
+			failed = TRUE;
+		}
+		stream->ftr = dcrypt_ctx_hmac_get_digest_length(stream->ctx_mac);
+		stream->use_mac = TRUE;
+	} else if ((stream->flags & IO_STREAM_ENC_INTEGRITY_AEAD) == IO_STREAM_ENC_INTEGRITY_AEAD) {
+		dcrypt_ctx_sym_set_aad(stream->ctx_sym, ptr, tagsize);
+		stream->ftr = tagsize;
+		stream->use_mac = TRUE;
+	} else {
+		stream->use_mac = FALSE;
+	}
+	/* destroy private key data */
+	safe_memset(buffer_get_modifiable_data(keydata, 0), 0, keydata->used);
+	buffer_set_used_size(keydata, 0);
+	return failed ? -1 : 1;
+}
+
+static
+ssize_t i_stream_decrypt_read_header(struct decrypt_istream *stream,
+	const unsigned char *data, size_t mlen)
+{
+	const char *error;
+	const unsigned char *end = data + mlen;
+
+	/* check magic */
+	if (mlen < sizeof(IOSTREAM_CRYPT_MAGIC))
+		return 0;
+	if (memcmp(data, IOSTREAM_CRYPT_MAGIC, sizeof(IOSTREAM_CRYPT_MAGIC)) != 0) {
+		io_stream_set_error(&stream->istream.iostream, "Invalid magic");
+		return -1;
+	}
+	data += sizeof(IOSTREAM_CRYPT_MAGIC);
+
+	if (data >= end)
+		return 0; /* read more? */
+
+	/* check version */
+	if (*data == '\x01') {
+		stream->format = DECRYPT_FORMAT_V1;
+		return i_stream_decrypt_read_header_v1(stream, data+1, end - (data+1));
+	} else if (*data != '\x02') {
+		io_stream_set_error(&stream->istream.iostream, "Unsupported encrypted data 0x%02x", *data);
+		return -1;
+	}
+
+	stream->format = DECRYPT_FORMAT_V2;
+
+	data++;
+
+	/* read flags */
+	uint32_t flags;
+	if (!get_msb32(&data, end, &flags))
+		return 0;
+	stream->flags = flags;
+
+	/* get the total length of header */
+	uint32_t hdr_len;
+	if (!get_msb32(&data, end, &hdr_len))
+		return 0;
+	if ((size_t)(end-data) < hdr_len)
+		return 0;
+
+	int ret;
+	if ((ret = i_stream_decrypt_header_contents(stream, data, hdr_len)) < 0)
+		return -1;
+	else if (ret == 0) {
+		io_stream_set_error(&stream->istream.iostream, "Decryption error: truncate header length");
+		return -1;
+	}
+	stream->initialized = TRUE;
+
+	/* if it all went well, try to initialize decryption context */
+	if (!dcrypt_ctx_sym_init(stream->ctx_sym, &error)) {
+		io_stream_set_error(&stream->istream.iostream, "Decryption init error: %s", error);
+		return -1;
+	}
+	return hdr_len;
+}
+
+static ssize_t
+i_stream_decrypt_read(struct istream_private *stream)
+{
+	struct decrypt_istream *dstream =
+		(struct decrypt_istream *)stream;
+	const unsigned char *data;
+	size_t size, decrypt_size;
+	const char *error = NULL;
+	int ret;
+	bool check_mac = FALSE;
+
+	/* not if it's broken */
+	if (stream->istream.stream_errno != 0)
+		return -1;
+
+	for (;;) {
+		/* remove skipped data from buffer */
+		if (stream->skip > 0) {
+			i_assert(stream->skip <= dstream->buf->used);
+			buffer_delete(dstream->buf, 0, stream->skip);
+			stream->pos -= stream->skip;
+			stream->skip = 0;
+		}
+
+		stream->buffer = dstream->buf->data;
+
+		i_assert(stream->pos <= dstream->buf->used);
+		if (stream->pos >= dstream->istream.max_buffer_size) {
+			/* stream buffer still at maximum */
+			return -2;
+		}
+
+		/* if something is already decrypted, return as much of it as
+		   we can */
+		if (dstream->buf->used > 0) {
+			size_t new_pos, bytes;
+
+			/* only return up to max_buffer_size bytes, even when buffer
+			   actually has more, as not to confuse the caller */
+			if (dstream->buf->used <= dstream->istream.max_buffer_size) {
+				new_pos = dstream->buf->used;
+				if (dstream->finalized)
+					stream->istream.eof = TRUE;
+			} else {
+				new_pos = dstream->istream.max_buffer_size;
+			}
+
+			bytes = new_pos - stream->pos;
+			stream->pos = new_pos;
+			return (ssize_t)bytes;
+		}
+		if (dstream->finalized) {
+			/* all data decrypted */
+			stream->istream.eof = TRUE;
+			return -1;
+		}
+		/* need to read more input */
+		ret = i_stream_read(stream->parent);
+		if (ret == 0)
+			return 0;
+		data = i_stream_get_data(stream->parent, &size);
+
+		if (ret == -1 && (size == 0 || stream->parent->stream_errno != 0)) {
+			stream->istream.stream_errno = stream->parent->stream_errno;
+			if (stream->istream.stream_errno != 0)
+				return -1;
+
+			if (!dstream->initialized) {
+				io_stream_set_error(&stream->iostream,
+					"Decryption error: %s",
+					"Input truncated in decryption header");
+				stream->istream.stream_errno = EINVAL;
+				return -1;
+			}
+
+			/* final block */
+			if (dcrypt_ctx_sym_final(dstream->ctx_sym,
+				dstream->buf, &error)) {
+				dstream->finalized = TRUE;
+				continue;
+			}
+			io_stream_set_error(&stream->iostream,
+				"MAC error: %s", error);
+			stream->istream.stream_errno = EINVAL;
+			return -1;
+		}
+
+		if (!dstream->initialized) {
+			ssize_t hret;
+			if ((hret=i_stream_decrypt_read_header
+				(dstream, data, size)) <= 0) {
+				if (hret < 0) {
+					stream->istream.stream_errno = EINVAL;
+				}
+				return hret;
+			}
+			i_stream_skip(stream->parent, hret);
+			data = i_stream_get_data(stream->parent, &size);
+		}
+		decrypt_size = size;
+
+		if (dstream->use_mac) {
+			if (stream->parent->eof) {
+				if (decrypt_size < dstream->ftr) {
+					io_stream_set_error(&stream->iostream,
+						"Decryption error: footer is longer than data");
+					stream->istream.stream_errno = EINVAL;
+					return -1;
+				}
+				check_mac = TRUE;
+			} else {
+				/* ignore footer's length of data until we
+				   reach EOF */
+				size -= dstream->ftr;
+			}
+			decrypt_size -= dstream->ftr;
+			if ((dstream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) {
+				if (!dcrypt_ctx_hmac_update(dstream->ctx_mac,
+				    data, decrypt_size, &error)) {
+					io_stream_set_error(&stream->iostream,
+						"MAC error: %s", error);
+					return -1;
+				}
+			}
+		}
+
+		if (check_mac) {
+			if ((dstream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) {
+				unsigned char dgst[dcrypt_ctx_hmac_get_digest_length(dstream->ctx_mac)];
+				buffer_t db;
+				buffer_create_from_data(&db, dgst, sizeof(dgst));
+				if (!dcrypt_ctx_hmac_final(dstream->ctx_mac, &db, &error)) {
+					io_stream_set_error(&stream->iostream,
+						"Cannot verify MAC: %s", error);
+				}
+				if (memcmp(dgst, data + decrypt_size, dcrypt_ctx_hmac_get_digest_length(dstream->ctx_mac)) != 0) {
+					io_stream_set_error(&stream->iostream,
+						"Cannot verify MAC: mismatch");
+					return -1;
+				}
+			} else if ((dstream->flags & IO_STREAM_ENC_INTEGRITY_AEAD) == IO_STREAM_ENC_INTEGRITY_AEAD) {
+				dcrypt_ctx_sym_set_tag(dstream->ctx_sym, data + decrypt_size, dstream->ftr);
+			}
+		}
+
+		if (!dcrypt_ctx_sym_update(dstream->ctx_sym,
+		    data, decrypt_size, dstream->buf, &error)) {
+			io_stream_set_error(&stream->iostream,
+				"Decryption error: %s", error);
+			stream->istream.stream_errno = EINVAL;
+			return -1;
+		}
+		i_stream_skip(stream->parent, size);
+	}
+}
+
+static
+void i_stream_decrypt_close(struct iostream_private *stream,
+				   bool close_parent)
+{
+	struct decrypt_istream *dstream =
+		(struct decrypt_istream *)stream;
+
+	if (close_parent)
+		i_stream_close(dstream->istream.parent);
+}
+
+static
+void i_stream_decrypt_destroy(struct iostream_private *stream)
+{
+	struct decrypt_istream *dstream =
+		(struct decrypt_istream *)stream;
+
+	if (dstream->buf != NULL)
+		buffer_free(&dstream->buf);
+	if (dstream->iv != NULL)
+		i_free_and_null(dstream->iv);
+	if (dstream->ctx_sym != NULL)
+		dcrypt_ctx_sym_destroy(&(dstream->ctx_sym));
+	if (dstream->ctx_mac != NULL)
+		dcrypt_ctx_hmac_destroy(&(dstream->ctx_mac));
+
+	i_stream_unref(&(dstream->istream.parent));
+}
+
+static
+struct decrypt_istream *i_stream_create_decrypt_common(struct istream *input)
+{
+	struct decrypt_istream *dstream;
+
+	dstream = i_new(struct decrypt_istream, 1);
+	dstream->istream.max_buffer_size = input->real_stream->max_buffer_size;
+	dstream->istream.read = i_stream_decrypt_read;
+	dstream->istream.iostream.close = i_stream_decrypt_close;
+	dstream->istream.iostream.destroy = i_stream_decrypt_destroy;
+
+	dstream->istream.istream.readable_fd = FALSE;
+	dstream->istream.istream.blocking = TRUE;
+	dstream->istream.istream.seekable = FALSE;
+
+	dstream->buf = buffer_create_dynamic(default_pool, 128);
+
+	(void)i_stream_create(&dstream->istream, input,
+			      i_stream_get_fd(input));
+	return dstream;
+}
+
+struct istream *
+i_stream_create_decrypt(struct istream *input, struct dcrypt_private_key *priv_key)
+{
+	struct decrypt_istream *dstream;
+
+	dstream = i_stream_create_decrypt_common(input);
+	dstream->priv_key = priv_key;
+	return &dstream->istream.istream;
+}
+
+struct istream *
+i_stream_create_sym_decrypt(struct istream *input, struct dcrypt_context_symmetric *ctx)
+{
+	const char *error;
+	int ec;
+	struct decrypt_istream *dstream;
+	dstream = i_stream_create_decrypt_common(input);
+	dstream->use_mac = FALSE;
+	dstream->initialized = TRUE;
+
+	if (!dcrypt_ctx_sym_init(ctx, &error)) ec = -1;
+	else ec = 0;
+
+	dstream->ctx_sym = ctx;
+
+	if (ec != 0) {
+		io_stream_set_error(&dstream->istream.iostream, "Cannot initialize decryption: %s", error);
+		dstream->istream.istream.stream_errno = EINVAL;
+	};
+
+	return &dstream->istream.istream;
+}
+
+struct istream *
+i_stream_create_decrypt_callback(struct istream *input,
+				 i_stream_decrypt_get_key_callback_t *callback,
+				 void *context)
+{
+	struct decrypt_istream *dstream;
+
+	i_assert(callback != NULL);
+
+	dstream = i_stream_create_decrypt_common(input);
+	dstream->key_callback = callback;
+	dstream->key_context = context;
+	return &dstream->istream.istream;
+}
diff --git a/src/lib-dcrypt/istream-decrypt.h b/src/lib-dcrypt/istream-decrypt.h
new file mode 100644
index 00000000000..713afbbcd42
--- /dev/null
+++ b/src/lib-dcrypt/istream-decrypt.h
@@ -0,0 +1,30 @@
+#ifndef ISTREAM_DECRYPT_H
+#define ISTREAM_DECRYPT_H
+
+struct dcrypt_private_key;
+struct dcrypt_context_symmetric;
+
+/* Look for a private key for a specified public key digest and set it to
+   priv_key_r. Returns 1 if ok, 0 if key doesn't exist, -1 on internal error. */
+typedef int
+i_stream_decrypt_get_key_callback_t(const unsigned char *pubkey_digest,
+				    struct dcrypt_private_key **priv_key_r,
+				    const char **error_r, void *context);
+
+struct istream *
+i_stream_create_decrypt(struct istream *input, struct dcrypt_private_key *priv_key);
+
+/* create stream for reading plain encrypted data with no header or MAC.
+   do not call dcrypt_ctx_sym_init
+ */
+struct istream *
+i_stream_create_sym_decrypt(struct istream *input, struct dcrypt_context_symmetric *ctx);
+
+
+/* Decrypt the istream. When a private key is needed, the callback will be
+   called. This allows using multiple private keys for different mails. */
+struct istream *
+i_stream_create_decrypt_callback(struct istream *input,
+				 i_stream_decrypt_get_key_callback_t *callback,
+				 void *context);
+#endif
diff --git a/src/lib-dcrypt/ostream-encrypt.c b/src/lib-dcrypt/ostream-encrypt.c
new file mode 100644
index 00000000000..70e69d3fb7b
--- /dev/null
+++ b/src/lib-dcrypt/ostream-encrypt.c
@@ -0,0 +1,686 @@
+/* file truct dcrypt_public_keyyntax
+ * magic (14 bytes)
+ * version (1 bytes)
+ * flags (4 bytes)
+ * size of header (4 bytes)
+ * sha1 of key id (20 bytes)
+ * cipher oid
+ * mac oid
+ * rounds (4 bytes)
+ * key data size (4 bytes)
+ * key data
+ * cipher data
+ * mac data (mac specific bytes)
+ */
+
+#include "lib.h"
+#include "buffer.h"
+#include "randgen.h"
+#include "ostream-encrypt.h"
+#include "ostream-private.h"
+#include "hash-method.h"
+#include "sha2.h"
+#include "safe-memset.h"
+#include "dcrypt.h"
+#include "dcrypt-iostream-private.h"
+
+#include <arpa/inet.h>
+
+#define IO_STREAM_ENCRYPT_SEED_SIZE 32
+#define IO_STREAM_ENCRYPT_ROUNDS 2048
+
+struct encrypt_ostream {
+	struct ostream_private ostream;
+
+	struct dcrypt_context_symmetric *ctx_sym;
+	struct dcrypt_context_hmac *ctx_mac;
+
+	enum io_stream_encrypt_flags flags;
+	struct dcrypt_public_key *pub;
+
+	unsigned char *key_data;
+	size_t key_data_len;
+
+	buffer_t *cipher_oid;
+	buffer_t *mac_oid;
+	size_t block_size;
+
+	bool finalized;
+	bool failed;
+	bool prefix_written;
+};
+
+static
+int o_stream_encrypt_send(struct encrypt_ostream *stream,
+			  const unsigned char *data, size_t size)
+{
+	ssize_t ec;
+
+	ec = o_stream_send(stream->ostream.parent, data, size);
+	if (ec == (ssize_t)size)
+		return 0;
+	else if (ec < 0) {
+		o_stream_copy_error_from_parent(&stream->ostream);
+		return -1;
+	} else {
+		io_stream_set_error(&stream->ostream.iostream,
+			"ostream-encrypt: Unexpectedly short write to parent stream");
+		stream->ostream.ostream.stream_errno = EINVAL;
+		return -1;
+	}
+}
+
+static
+int o_stream_encrypt_send_header_v1(struct encrypt_ostream *stream)
+{
+	unsigned char c;
+	unsigned short s;
+
+	i_assert(!stream->prefix_written);
+	stream->prefix_written = TRUE;
+
+	buffer_t *values = buffer_create_dynamic(pool_datastack_create(), 256);
+	buffer_append(values, IOSTREAM_CRYPT_MAGIC, sizeof(IOSTREAM_CRYPT_MAGIC));
+	/* version */
+	c = 1;
+	buffer_append(values, &c, 1);
+	/* header length including this and data written so far */
+	s = htons(stream->key_data_len);
+	buffer_append(values, &s, 2);
+	/* then write key data */
+	buffer_append(values, stream->key_data, stream->key_data_len);
+	i_free_and_null(stream->key_data);
+
+	/* then send it to stream */
+	return o_stream_encrypt_send(stream, values->data, values->used);
+}
+
+static
+int o_stream_encrypt_send_header_v2(struct encrypt_ostream *stream)
+{
+	unsigned char c;
+	unsigned int i;
+
+	i_assert(!stream->prefix_written);
+	stream->prefix_written = TRUE;
+
+	buffer_t *values = buffer_create_dynamic(pool_datastack_create(), 256);
+	buffer_append(values, IOSTREAM_CRYPT_MAGIC, sizeof(IOSTREAM_CRYPT_MAGIC));
+	c = 2;
+	buffer_append(values, &c, 1);
+	i = htonl(stream->flags);
+	buffer_append(values, &i, 4);
+	/* store total length of header
+	   9 = version + flags + length
+	   8 = rounds + key data length
+	   */
+	i = htonl(sizeof(IOSTREAM_CRYPT_MAGIC) + 9 + stream->cipher_oid->used + stream->mac_oid->used + 8 + stream->key_data_len);
+	buffer_append(values, &i, 4);
+
+	buffer_append_buf(values, stream->cipher_oid, 0, (size_t)-1);
+	buffer_append_buf(values, stream->mac_oid, 0, (size_t)-1);
+	i = htonl(IO_STREAM_ENCRYPT_ROUNDS);
+	buffer_append(values, &i, 4);
+	i = htonl(stream->key_data_len);
+	buffer_append(values, &i, 4);
+	buffer_append(values, stream->key_data, stream->key_data_len);
+	i_free_and_null(stream->key_data);
+
+	return o_stream_encrypt_send(stream, values->data, values->used);
+}
+
+static
+int o_stream_encrypt_keydata_create_v1(struct encrypt_ostream *stream)
+{
+	buffer_t *encrypted_key, *ephemeral_key, *secret, *res, buf;
+	const char *error = NULL;
+	const struct hash_method *hash = &hash_method_sha256;
+
+	/* various temporary buffers */
+	unsigned char seed[IO_STREAM_ENCRYPT_SEED_SIZE];
+	unsigned char pkhash[hash->digest_size];
+	unsigned char ekhash[hash->digest_size];
+	unsigned char hres[hash->digest_size];
+
+	unsigned char hctx[hash->context_size];
+
+	/* hash the public key first */
+	buffer_create_from_data(&buf, pkhash, sizeof(pkhash));
+	if (!dcrypt_key_id_public_old(stream->pub, &buf, &error)) {
+		io_stream_set_error(&stream->ostream.iostream, "Key hash failed: %s", error);
+		return -1;
+	}
+
+	/* hash the key base */
+	hash->init(hctx);
+	hash->loop(hctx, seed, sizeof(seed));
+	hash->result(hctx, ekhash);
+
+	ephemeral_key = buffer_create_dynamic(pool_datastack_create(), 256);
+	encrypted_key = buffer_create_dynamic(pool_datastack_create(), 256);
+	secret = buffer_create_dynamic(pool_datastack_create(), 256);
+
+	if (!dcrypt_ecdh_derive_secret_peer(stream->pub, ephemeral_key, secret, &error)) {
+		io_stream_set_error(&stream->ostream.iostream, "Cannot perform ECDH: %s", error);
+		return -1;
+	}
+
+	/* hash the secret data */
+	hash->init(hctx);
+	hash->loop(hctx, secret->data, secret->used);
+	hash->result(hctx, hres);
+	safe_memset(buffer_get_modifiable_data(secret, 0), 0, secret->used);
+
+	/* use it to encrypt the actual encryption key */
+	struct dcrypt_context_symmetric *dctx;
+	if (!dcrypt_ctx_sym_create("aes-256-ctr", DCRYPT_MODE_ENCRYPT, &dctx, &error)) {
+		io_stream_set_error(&stream->ostream.iostream, "Key encryption error: %s", error);
+		return -1;
+	}
+
+	random_fill(seed, sizeof(seed));
+	hash->init(hctx);
+	hash->loop(hctx, seed, sizeof(seed));
+	hash->result(hctx, ekhash);
+
+	int ec = 0;
+
+	/* NB! The old code was broken and used this kind of IV - it is not correct, but
+	   we need to stay compatible with old data */
+	dcrypt_ctx_sym_set_iv(dctx, (const unsigned char*)"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 16);
+	dcrypt_ctx_sym_set_key(dctx, hres, sizeof(hres));
+
+	if (!dcrypt_ctx_sym_init(dctx, &error) ||
+	    !dcrypt_ctx_sym_update(dctx, seed, sizeof(seed), encrypted_key, &error) ||
+	    !dcrypt_ctx_sym_final(dctx, encrypted_key, &error)) {
+		ec = -1;
+	}
+	dcrypt_ctx_sym_destroy(&dctx);
+
+	if (ec != 0) {
+		safe_memset(seed, 0, sizeof(seed));
+		io_stream_set_error(&stream->ostream.iostream, "Key encryption error: %s", error);
+		return -1;
+	}
+
+	if (ec == 0) {
+		/* same as above */
+		dcrypt_ctx_sym_set_iv(stream->ctx_sym, (const unsigned char*)"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00", 16);
+		dcrypt_ctx_sym_set_key(stream->ctx_sym, seed, sizeof(seed));
+	}
+	safe_memset(seed, 0, sizeof(seed));
+
+	if (ec != 0) {
+		io_stream_set_error(&stream->ostream.iostream, "Encryption init error: %s", error);
+		return -1;
+	}
+
+	if (!dcrypt_ctx_sym_init(stream->ctx_sym, &error)) {
+		io_stream_set_error(&stream->ostream.iostream, "Encryption init error: %s", error);
+		return -1;
+	}
+
+	res = buffer_create_dynamic(default_pool, 256);
+
+	/* ephemeral key */
+	unsigned short s;
+	s = htons(ephemeral_key->used);
+	buffer_append(res, &s, 2);
+	buffer_append(res, ephemeral_key->data, ephemeral_key->used);
+	/* public key hash */
+	s = htons(sizeof(pkhash));
+	buffer_append(res, &s, 2);
+	buffer_append(res, pkhash, sizeof(pkhash));
+	/* encrypted key hash */
+	s = htons(sizeof(ekhash));
+	buffer_append(res, &s, 2);
+	buffer_append(res, ekhash, sizeof(ekhash));
+	/* encrypted key */
+	s = htons(encrypted_key->used);
+	buffer_append(res, &s, 2);
+	buffer_append(res, encrypted_key->data, encrypted_key->used);
+
+	stream->key_data_len = res->used;
+	stream->key_data = buffer_free_without_data(&res);
+
+	return 0;
+}
+
+static
+int o_stream_encrypt_key_for_pubkey_v2(struct encrypt_ostream *stream, const char *malg,
+	const unsigned char *key, size_t key_len, struct dcrypt_public_key *pubkey, buffer_t *res)
+{
+	enum dcrypt_key_type ktype;
+	const char *error;
+	buffer_t *encrypted_key, *ephemeral_key, *temp_key;
+
+	ephemeral_key = buffer_create_dynamic(pool_datastack_create(), 256);
+	encrypted_key = buffer_create_dynamic(pool_datastack_create(), 256);
+	temp_key = buffer_create_dynamic(pool_datastack_create(), 48);
+
+	dcrypt_key_type_public(pubkey, &ktype);
+
+	if (ktype == DCRYPT_KEY_RSA) {
+		/* encrypt key as R (as we don't need DH with RSA)*/
+		if (!dcrypt_rsa_encrypt(pubkey, key, key_len, encrypted_key, &error)) {
+			io_stream_set_error(&stream->ostream.iostream, "Cannot encrypt key data: %s", error);
+			return -1;
+		}
+	} else if (ktype == DCRYPT_KEY_EC) {
+		/* R = our ephemeral public key */
+		buffer_t *secret = buffer_create_dynamic(pool_datastack_create(), 256);
+
+		/* derive ephemeral key and shared secret */
+		if (!dcrypt_ecdh_derive_secret_peer(pubkey, ephemeral_key, secret, &error)) {
+			io_stream_set_error(&stream->ostream.iostream, "Cannot perform ECDH: %s", error);
+			return -1;
+		}
+
+		/* use shared secret and ephemeral key to generate encryption key/iv */
+		if (!dcrypt_pbkdf2(secret->data, secret->used, ephemeral_key->data, ephemeral_key->used,
+		    malg, IO_STREAM_ENCRYPT_ROUNDS, temp_key, 48, &error)) {
+			safe_memset(buffer_get_modifiable_data(secret, 0), 0, secret->used);
+			io_stream_set_error(&stream->ostream.iostream, "Cannot perform key encryption: %s", error);
+		}
+		safe_memset(buffer_get_modifiable_data(secret, 0), 0, secret->used);
+
+		/* encrypt key with shared secret */
+		struct dcrypt_context_symmetric *dctx;
+		if (!dcrypt_ctx_sym_create("AES-256-CBC", DCRYPT_MODE_ENCRYPT, &dctx, &error)) {
+			safe_memset(buffer_get_modifiable_data(temp_key, 0), 0, temp_key->used);
+			io_stream_set_error(&stream->ostream.iostream, "Cannot perform key encryption: %s", error);
+			return -1;
+		}
+
+		const unsigned char *ptr = temp_key->data;
+		i_assert(temp_key->used == 48);
+
+		dcrypt_ctx_sym_set_key(dctx, ptr, 32);
+		dcrypt_ctx_sym_set_iv(dctx, ptr+32, 16);
+		safe_memset(buffer_get_modifiable_data(temp_key, 0), 0, temp_key->used);
+
+		int ec = 0;
+		if (!dcrypt_ctx_sym_init(dctx, &error) ||
+		    !dcrypt_ctx_sym_update(dctx, key, key_len, encrypted_key, &error) ||
+		    !dcrypt_ctx_sym_final(dctx, encrypted_key, &error)) {
+			io_stream_set_error(&stream->ostream.iostream, "Cannot perform key encryption: %s", error);
+			ec = -1;
+		}
+
+		dcrypt_ctx_sym_destroy(&dctx);
+		if (ec != 0) return ec;
+	} else {
+		io_stream_set_error(&stream->ostream.iostream, "Unsupported key type");
+		return -1;
+	}
+
+	/* store key type */
+	char kt = ktype;
+	buffer_append(res, &kt, 1);
+	/* store hash of public key as ID */
+	dcrypt_key_id_public(stream->pub, "sha256", res, NULL);
+	/* store ephemeral key (if present) */
+	unsigned int val = htonl(ephemeral_key->used);
+	buffer_append(res, &val, 4);
+	buffer_append_buf(res, ephemeral_key, 0, (size_t)-1);
+	/* store encrypted key */
+	val = htonl(encrypted_key->used);
+	buffer_append(res, &val, 4);
+	buffer_append_buf(res, encrypted_key, 0, (size_t)-1);
+
+	return 0;
+}
+
+static
+int o_stream_encrypt_keydata_create_v2(struct encrypt_ostream *stream, const char *malg)
+{
+	const struct hash_method *hash = hash_method_lookup(malg);
+	const char *error;
+	size_t tagsize;
+	const unsigned char *ptr;
+	size_t kl;
+	unsigned int val;
+
+	buffer_t *keydata, *res;
+
+	if (hash == NULL) {
+		io_stream_set_error(&stream->ostream.iostream,
+			"Encryption init error: Hash algorithm '%s' not supported", malg);
+		return -1;
+	}
+
+	/* key data length for internal use */
+	if ((stream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) {
+		tagsize = IOSTREAM_TAG_SIZE; 
+	} else if ((stream->flags & IO_STREAM_ENC_INTEGRITY_AEAD) == IO_STREAM_ENC_INTEGRITY_AEAD) {
+		tagsize = IOSTREAM_TAG_SIZE;
+	} else {
+		/* do not include MAC */
+		tagsize = 0;
+	}
+
+	/* generate keydata length of random data for key/iv/mac */
+	kl = dcrypt_ctx_sym_get_key_length(stream->ctx_sym) + dcrypt_ctx_sym_get_iv_length(stream->ctx_sym) + tagsize;
+	keydata = buffer_create_dynamic(pool_datastack_create(), kl);
+	random_fill(buffer_append_space_unsafe(keydata, kl), kl);
+	buffer_set_used_size(keydata, kl);
+	ptr = keydata->data;
+
+	res = buffer_create_dynamic(default_pool, 256);
+
+	/* store number of public key(s) */
+	buffer_append(res, "\1", 1); /* one key for now */
+
+	/* we can do multiple keys at this point, but do it only once now */
+	if (o_stream_encrypt_key_for_pubkey_v2(stream, malg, ptr, kl, stream->pub, res) != 0) {
+		buffer_free(&res);
+		return -1;
+	}
+
+	/* create hash of the key data */
+	unsigned char hctx[hash->context_size];
+	unsigned char hres[hash->digest_size];
+	hash->init(hctx);
+	hash->loop(hctx, ptr, kl);
+	hash->result(hctx, hres);
+
+	for(int i = 1; i < 2049; i++) {
+		uint32_t i_msb = htonl(i);
+
+		hash->init(hctx);
+		hash->loop(hctx, hres, sizeof(hres));
+		hash->loop(hctx, &i_msb, sizeof(i_msb));
+		hash->result(hctx, hres);
+	}
+
+	/* store key data hash */
+	val = htonl(sizeof(hres));
+	buffer_append(res, &val, 4);
+	buffer_append(res, hres, sizeof(hres));
+
+	/* pick up key data that goes into stream */
+	stream->key_data_len = res->used;
+	stream->key_data = buffer_free_without_data(&res);
+
+	/* prime contexts */
+	dcrypt_ctx_sym_set_key(stream->ctx_sym, ptr, dcrypt_ctx_sym_get_key_length(stream->ctx_sym));
+	ptr += dcrypt_ctx_sym_get_key_length(stream->ctx_sym);
+	dcrypt_ctx_sym_set_iv(stream->ctx_sym, ptr, dcrypt_ctx_sym_get_iv_length(stream->ctx_sym));
+	ptr += dcrypt_ctx_sym_get_iv_length(stream->ctx_sym);
+
+	if ((stream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) {
+		dcrypt_ctx_hmac_set_key(stream->ctx_mac, ptr, tagsize);
+		dcrypt_ctx_hmac_init(stream->ctx_mac, &error);
+	} else if ((stream->flags & IO_STREAM_ENC_INTEGRITY_AEAD) == IO_STREAM_ENC_INTEGRITY_AEAD) {
+		dcrypt_ctx_sym_set_aad(stream->ctx_sym, ptr, tagsize);
+	}
+
+	/* clear out private key data */
+	safe_memset(buffer_get_modifiable_data(keydata, 0), 0, keydata->used);
+
+	if (!dcrypt_ctx_sym_init(stream->ctx_sym, &error)) {
+		io_stream_set_error(&stream->ostream.iostream, "Encryption init error: %s", error);
+		return -1;
+	}
+	return 0;
+}
+
+static
+ssize_t o_stream_encrypt_sendv(struct ostream_private *stream,
+		       const struct const_iovec *iov, unsigned int iov_count)
+{
+	struct encrypt_ostream *estream = (struct encrypt_ostream *)stream;
+	const char *error;
+	ssize_t ec,total = 0;
+
+	/* not if finalized */
+	i_assert(!estream->finalized);
+
+	/* write prefix */
+	if (!estream->prefix_written) {
+		T_BEGIN {
+			if ((estream->flags & IO_STREAM_ENC_VERSION_1) == IO_STREAM_ENC_VERSION_1)
+				ec = o_stream_encrypt_send_header_v1(estream);
+			else
+				ec = o_stream_encrypt_send_header_v2(estream);
+		} T_END;
+		if (ec < 0) {
+			return -1;
+		}
+	}
+
+	/* buffer for encrypted data */
+	unsigned char ciphertext[IO_BLOCK_SIZE];
+	buffer_t buf;
+	buffer_create_from_data(&buf, ciphertext, sizeof(ciphertext));
+
+	/* encrypt & send all blocks of data at max ciphertext buffer's length */
+	for(unsigned int i = 0; i < iov_count; i++) {
+		size_t bl, off = 0, len = iov[i].iov_len;
+		const unsigned char *ptr = iov[i].iov_base;
+		while(len > 0) {
+			buffer_set_used_size(&buf, 0);
+			/* update can emite twice the size of input */
+			bl = I_MIN(sizeof(ciphertext)/2, len);
+
+			if (!dcrypt_ctx_sym_update(estream->ctx_sym, ptr + off, bl, &buf, &error)) {
+				io_stream_set_error(&stream->iostream, "Encryption failure: %s", error);
+				return -1;
+			}
+			if ((estream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) {
+				/* update mac */
+				if (!dcrypt_ctx_hmac_update(estream->ctx_mac, buf.data, buf.used, &error)) {
+					io_stream_set_error(&stream->iostream, "MAC failure: %s", error);
+					return -1;
+				}
+			}
+
+			/* hopefully upstream can accomondate */
+			if (o_stream_encrypt_send(estream, buf.data, buf.used) < 0) {
+				return -1;
+			}
+
+			len -= bl;
+			off += bl;
+			total += bl;
+		}
+	}
+
+	stream->ostream.offset += total;
+	return total;
+}
+
+static
+int o_stream_encrypt_flush(struct ostream_private *stream)
+{
+	const char *error;
+	struct encrypt_ostream *estream = (struct encrypt_ostream *)stream;
+
+	i_assert(!estream->finalized);
+	estream->finalized = TRUE;
+
+	/* acquire last block */
+	buffer_t *buf = buffer_create_dynamic(pool_datastack_create(), dcrypt_ctx_sym_get_block_size(estream->ctx_sym));
+	if (!dcrypt_ctx_sym_final(estream->ctx_sym, buf, &error)) {
+		io_stream_set_error(&estream->ostream.iostream, "Encryption failure: %s", error);
+		return -1;
+	}
+	/* sometimes final does not emit anything */
+	if (buf->used > 0) {
+		/* update mac */
+		if (((estream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC)) {
+			if (!dcrypt_ctx_hmac_update(estream->ctx_mac, buf->data, buf->used, &error)) {
+				io_stream_set_error(&estream->ostream.iostream, "MAC failure: %s", error);
+				return -1;
+			}
+		}
+		if (o_stream_encrypt_send(estream, buf->data, buf->used) < 0) {
+			return -1;
+		}
+	}
+
+	/* write last mac bytes */
+	buffer_set_used_size(buf, 0);
+	if ((estream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) {
+		if (!dcrypt_ctx_hmac_final(estream->ctx_mac, buf, &error)) {
+			io_stream_set_error(&estream->ostream.iostream, "MAC failure: %s", error);
+			return -1;
+		}
+	} else if ((estream->flags & IO_STREAM_ENC_INTEGRITY_AEAD) == IO_STREAM_ENC_INTEGRITY_AEAD) {
+		dcrypt_ctx_sym_get_tag(estream->ctx_sym, buf);
+		i_assert(buf->used > 0);
+	}
+	if (buf->used > 0 && o_stream_encrypt_send(estream, buf->data, buf->used) < 0) {
+		return -1;
+	}
+
+	/* flush parent */
+	return o_stream_flush(stream->parent);
+}
+
+static
+void o_stream_encrypt_close(struct iostream_private *stream,
+			    bool close_parent)
+{
+	struct encrypt_ostream *estream = (struct encrypt_ostream *)stream;
+	if (estream->ctx_sym != NULL && !estream->finalized &&
+	    estream->ostream.ostream.stream_errno == 0)
+		o_stream_encrypt_flush(&estream->ostream);
+	if (close_parent) {
+		o_stream_close(estream->ostream.parent);
+	}
+}
+
+static
+void o_stream_encrypt_destroy(struct iostream_private *stream)
+{
+	struct encrypt_ostream *estream = (struct encrypt_ostream *)stream;
+	/* release resources */
+	if (estream->ctx_sym != NULL) dcrypt_ctx_sym_destroy(&(estream->ctx_sym));
+	if (estream->ctx_mac != NULL) dcrypt_ctx_hmac_destroy(&(estream->ctx_mac));
+	if (estream->key_data != NULL) i_free(estream->key_data);
+	if (estream->cipher_oid != NULL) buffer_free(&(estream->cipher_oid));
+	if (estream->mac_oid != NULL) buffer_free(&(estream->mac_oid));
+
+	o_stream_unref(&estream->ostream.parent);
+}
+
+static
+int o_stream_encrypt_init(struct encrypt_ostream *estream, const char *algorithm)
+{
+	const char *error;
+	char *calg, *malg;
+
+	if ((estream->flags & IO_STREAM_ENC_VERSION_1) == IO_STREAM_ENC_VERSION_1) {
+		if (!dcrypt_ctx_sym_create("AES-256-CTR", DCRYPT_MODE_ENCRYPT, &(estream->ctx_sym), &error)) {
+			io_stream_set_error(&estream->ostream.iostream, "Cannot create ostream-encrypt: %s", error);
+			return -1;
+		}
+		estream->flags |= IO_STREAM_ENC_INTEGRITY_NONE; /* disable MAC */
+		/* then do keying */
+		return o_stream_encrypt_keydata_create_v1(estream);
+	} else {
+		calg = t_strdup_noconst(algorithm);
+		malg = strrchr(calg, '-');
+
+		if (malg == NULL) {
+			io_stream_set_error(&estream->ostream.iostream, "Invalid algorithm (must be cipher-mac)");
+			return -1;
+		}
+		(*malg++) = '\0';
+
+		if (!dcrypt_ctx_sym_create(calg, DCRYPT_MODE_ENCRYPT, &(estream->ctx_sym), &error)) {
+			io_stream_set_error(&estream->ostream.iostream, "Cannot create ostream-encrypt: %s", error);
+			return -1;
+		}
+
+		/* create cipher and mac context, take note of OIDs */
+		estream->cipher_oid = buffer_create_dynamic(default_pool, 12);
+		estream->block_size = dcrypt_ctx_sym_get_block_size(estream->ctx_sym);
+		if (!dcrypt_name2oid(calg, estream->cipher_oid, &error)) {
+			io_stream_set_error(&estream->ostream.iostream, "Cannot create ostream-encrypt: %s", error);
+			return -1;
+		}
+
+		/* mac context is optional */
+		if ((estream->flags & IO_STREAM_ENC_INTEGRITY_HMAC) == IO_STREAM_ENC_INTEGRITY_HMAC) {
+			if (!dcrypt_ctx_hmac_create(malg, &(estream->ctx_mac), &error)) {
+				io_stream_set_error(&estream->ostream.iostream, "Cannot create ostream-encrypt: %s", error);
+				return -1;
+			}
+		}
+
+		estream->mac_oid = buffer_create_dynamic(default_pool, 12);
+		if (!dcrypt_name2oid(malg, estream->mac_oid, &error)) {
+			io_stream_set_error(&estream->ostream.iostream, "Cannot create ostream-encrypt: %s", error);
+			return -1;
+		}
+
+		/* MAC algoritm is used for PBKDF2 and keydata hashing */
+		return o_stream_encrypt_keydata_create_v2(estream, malg);
+	}
+}
+
+static
+struct encrypt_ostream *
+o_stream_create_encrypt_common(enum io_stream_encrypt_flags flags)
+{
+	struct encrypt_ostream *estream;
+
+	estream = i_new(struct encrypt_ostream, 1);
+	estream->ostream.sendv = o_stream_encrypt_sendv;
+	estream->ostream.flush = o_stream_encrypt_flush;
+	estream->ostream.iostream.close = o_stream_encrypt_close;
+	estream->ostream.iostream.destroy = o_stream_encrypt_destroy;
+
+	estream->flags = flags;
+
+	return estream;
+}
+
+struct ostream *
+o_stream_create_encrypt(struct ostream *output, const char *algorithm,
+	struct dcrypt_public_key *box_pub, enum io_stream_encrypt_flags flags)
+{
+	struct encrypt_ostream *estream = o_stream_create_encrypt_common(flags);
+	int ec;
+
+	estream->pub = box_pub;
+
+	T_BEGIN {
+		ec = o_stream_encrypt_init(estream, algorithm);
+	} T_END;
+
+	struct ostream *os = o_stream_create(&estream->ostream, output,
+			       o_stream_get_fd(output));
+
+	if (ec != 0) {
+		os->stream_errno = EINVAL;
+	}
+
+	return os;
+}
+
+struct ostream *
+o_stream_create_sym_encrypt(struct ostream *output, struct dcrypt_context_symmetric *ctx)
+{
+	struct encrypt_ostream *estream = o_stream_create_encrypt_common(IO_STREAM_ENC_INTEGRITY_NONE);
+	const char *error;
+	int ec;
+
+	estream->prefix_written = TRUE;
+
+	if (!dcrypt_ctx_sym_init(estream->ctx_sym, &error)) ec = -1;
+	else ec = 0;
+
+	estream->ctx_sym = ctx;
+
+	struct ostream *os = o_stream_create(&estream->ostream, output,
+		 o_stream_get_fd(output));
+	if (ec != 0) {
+		io_stream_set_error(&estream->ostream.iostream, "Could not initialize stream: %s", error);
+		os->stream_errno = EINVAL;
+	}
+
+	return os;
+}
diff --git a/src/lib-dcrypt/ostream-encrypt.h b/src/lib-dcrypt/ostream-encrypt.h
new file mode 100644
index 00000000000..b6f628c2b3e
--- /dev/null
+++ b/src/lib-dcrypt/ostream-encrypt.h
@@ -0,0 +1,38 @@
+#ifndef OSTREAM_ENCRYPT_H
+#define OSTREAM_ENCRYPT_H
+
+struct dcrypt_public_key;
+struct dcrypt_context_symmetric;
+
+/**
+ * algorithm is in form AES-256-CBC-SHA1, recommended
+ * AES-256-GCM-SHA256
+ *
+ * Algorithms (both crypto and digest) *MUST* have OID to use it.
+ *
+ */
+
+enum io_stream_encrypt_flags {
+	IO_STREAM_ENC_INTEGRITY_HMAC = 0x1,
+	IO_STREAM_ENC_INTEGRITY_AEAD = 0x2,
+	IO_STREAM_ENC_INTEGRITY_NONE = 0x4,
+	IO_STREAM_ENC_VERSION_1      = 0x8,
+};
+
+struct ostream *
+o_stream_create_encrypt(struct ostream *output,
+	const char *algorithm,
+	struct dcrypt_public_key *box_pub,
+	enum io_stream_encrypt_flags flags);
+
+/* create context for performing encryption with
+   preset crypto context. do not call ctx_sym_init.
+
+   no header or mac is written, just plain crypto
+   data.
+ */
+struct ostream *
+o_stream_create_sym_encrypt(struct ostream *output,
+	struct dcrypt_context_symmetric *ctx);
+
+#endif
diff --git a/src/lib-dcrypt/sample-v1.bin b/src/lib-dcrypt/sample-v1.bin
new file mode 100644
index 0000000000000000000000000000000000000000..e43e95b518a27a95de1758f398e6fc4282c1c046
GIT binary patch
literal 2281
zcmV<F2p0E4Qdv+`MMMJ!0RXK4LjnNIinwFHk95zWe9<R=Qyr~n7%-ul=c+|fWk-6a
zA!i*j-FeQRqq#D3!Td@L+aPp@#g*b4y6QCneN%!dnpO7v03dvt5IO2DPwjK)9n=5A
z`XcH|C^acLt=@b1>V=B{`cD8LS}({Qo$VH8sD<lPkXkT-%GkijeH|ZSIe0oj(gU8O
z03eFO^ydr8<1<p7d@hwrWVkv(qC~Pf-lP_-f=ZDN07?J=`?hjEMmXTC)_G8NFN_Uw
zvaun3ys`0qm2c;!`|5sZPl)$wL9V5LT!)d>>h+-2_n#JU5_0%c2wJ_9jKj1iYf3n7
zig-Qpy5xGN6*-`4BSQqoMEoII#Uy1TL(6kW;j0=1VNc!C8tpyGP%^D73lLi#rzZ?u
zaR9BM0hKk_5&kBHcx;Vi)Wav&5FB3^m3GvQ<k376m-1-G?@q0=A)9JJjXi3*-Ox6W
z@{h~e(iQ476*ZFaZ4p*gtf*RA6a9r!lz3vpmIo#4Uww15+G%Bvb1|=$h<qujWxCv0
z$Lkl*B(RQ$$pfoQa+3LvdHGRGZydPz6@?h3c+EkS!nec>>Tw_jTLgOm1~q)x)V54U
zWw!6+i*X^B0K6YBsLHASEt7dHDVw+XnlOz7B~F|)B?3Ut5vyhI?-TzHu(tzDyMjIG
zPmUd9M@)~tpacp^JA^Q-w*i5VYRSdNwM;Lz571<q+uD7A2e@?j9m|1Pygyx~)Sv}R
zY^1sXR#8$?xk$?8_|ubt10kTM(ztBe6K%`Wtp-4DKpG{JJV&;wegEVkA!0<d+fWhN
z@HfEic2h_Gp1rY~<PI6KsVjlVDjmUCWIdPpclriZOT_PX?UVvsm%~{xaO!}e-%o`(
zI9`}}CDbE7Xo$_0%TSf7ur?*pMni$zZT|Wo{yN*}26^+H>t?ML`{Nj58~Eumu4bEz
z+-Lv1EmICQF>*7Y2|gJM4uX#RtbIG?VhdWTbUNTNMKY$bl+WB&gKvbfLQTr24bAL$
zjUph{*9Un7vPAECu`0ib3&D%C`#vU@)o<x>&2S1{kr_;3UH>{^7gC=m=79)bln0XP
z`35K*@{%VsTib)ua0P7DTGjG$WkUQ+Wp0soiXS!042_SQ){t>-{`mTU0ugY(7g*`1
zdrVC-&Me<Qgxa(xS%lu>m_!DRD_Gh=04QsU&Bf+F+VF6ihkX1>-1N;1;2qT+SJ5Yd
zd@u}mZ^)nCh@x=2v^br5vT=tqutuk9P7w4}0u506y)zBVZb@E;!1fXWc8J4b&#k)<
znypK#@?yMp5yCYC1W+1YL+{yPM14ya64|g*H4iQYv1HE-u<*nqopehyUEJwGQah6O
zufO(x(7#am3>oVTe_&GuB0j%W`m+p@V_E?UQ!0>+rR%~U@pbaFCsHrZB&Ij+uPGP#
zPSEAMn>4-)5U&=$np+ljkzY6)1uIhIogJ-N|3Fy%MA@HEK#Qi%x!9H*er2GLWp=lc
zCt|k#EDWTz6cXe<Op<&C`xbcO%&1C3BO9(@L_$)b&hSaCzQm6y!X8VuS*H0Bl@b;T
z;-H<O(#mA~>`Jvrn<P#^7I#>Ny#hP((>yB#_9!`@?F%<WK%z@2)BJr-nN%vlS^wqF
za27Rw(DXnkR#^-~FT*ZvVv==ZN!YnX3_4IemtAY5s(>NHW6@o?>gDK<mjRtS24n+;
z3>vYd66P3KGu9TbkHl<(t4Vw*B!}iam)va30KN0^Mquk^DwFuJb<Etb5|S9U6;1*;
zfB3aaD2IDY3F+O(eyr6=rHD^>1|x1H#Q>=&eQ>a+TlFHtM)Rp(1~`(#lkhiO#j6n;
zfY_P)sSjTEshDcFJg=f-Sxd1HCUoK5cEHe_3|`?|5m+s}6O;B-AkIV8n;kgy5Dos|
zj77MC3Jxo}P+BL+Yo3`&A#SX?3+1Q2-P7<=?*}yk@4{?&rKFzITOmQ+`Z0GAXRuLL
zod&4d?~33pU8fSq9Y!GUYlpOL$YHG33hYBp&*)3(lSx0-W;pS|KE-7@NPY>!6B*w2
z!-C@&{mJRsSZ>XjE+>eH6sS2w#Zrc}q-6&dz<%EBm2U16p53*n)TfmjjYzQJX`@14
zrRXhz{wvlNB8N?o=Y<U*SNR)q5wWnfraNN*wk>!f-u8yR?ot3$HK3ZW&8cGSi-&EO
zKYL7X7Xp$6XQS({vWD>Akrg?pxposkeucPRSG=7QsE7!W)U$5pjhNm$6Qj_eFfK%%
z7xk!Eq+M|fhn$=Si2Au&rK-heQ;xR=6+=<&h0gG&?X=~hy1F)UQ>MK#v6mT{B}E5f
zfG2(s>ok#Uytn4unof%KrNk#8j0V>rKrxnOl3uTzJZVCs=^*L%F4xvffVvob3|q}8
z$PwQAIKq4Ej=ffKj7RYsz*Ttr>3HwH2cX_J32n;snEV0JD%_<Hyz*gs;~%H2s~4L;
zMDqY(IL12SrPpt6*iq;}7~eIGHc>apQiff$eskF=-c;7gh=uFdHA>S~JXJOK^n7$@
zIOhGaQ1`vmCVjQ<2YK>9vXJ>_F#TidYd~(dl7w7X=V^1!MZp#lmM>W720iE|tvDe+
zhE=h|UB-;>cS{PslFc#S#E8<tX@~9mhA7{~nPNtBG?e=;6}Z-x(8#S^nV6Ia=&VTn
ztFG%hh@g$Zp^q_=O$aAYH_o;VB-(328AwNxMwh{S4U~HHtbKhhP7Ip|fHT(a5GMa$
zLZ2uCdFdav2Ze4oSN0QD><cVj>8UApCuj|Dv0obERescI>k?p^N7p+V3DYh@K~Cwx
z27ec&vAJMZ)^5x+PtJUgW%S_V(%>K}eO~xH$JJHYi^*)2>Cn0AWVgKPTy9-O7*_aH
zr6(<DquLK~{hMYG7&6?qnjb~riEP~@f)k|raZLcpfQ_!NA}cDnS6JN(*jhJ}W*6Lz
zh>TcS*5ZM{Q_UWJ0h^6`wuwo5w5s$8N82=a`Fkg#>>qAO|2qg*{SF<<?=e9i*=*F^
D+SXiI

literal 0
HcmV?d00001

diff --git a/src/lib-dcrypt/sample-v2.bin b/src/lib-dcrypt/sample-v2.bin
new file mode 100755
index 0000000000000000000000000000000000000000..0ee99bba14d5255212d113b55bb4574d3350af9f
GIT binary patch
literal 13757
zcmV;uHA2cmQdv+`MMMJ!0ssI20ssI2&jtx#hDZTr0|Wss1_@w>NC9O71Ofp700;m8
z005r>0;?0!QI~D)t5j=eHBNSTf)_yDQ%iK5C|CAt_A9#G{Qv*}Ap>#&Lc0oD&VJbO
z_{WUK@6cXXXg`3X@ec5+{o<r&3}*lU05DkMk0y>5JW5#;SmeXWsD|#T9Dj2=n#&Mz
zrTo$CYW1dvH@yJ55_)iY49KjAId%X503aP`ZtI@NLG$yVs$heX5t{n(;0`s0JMvb`
z`Kx_)rVbCP%7P&5SYq9!Fb(caSL#eozL1KnSQgw9I(VY2kGCmcdVIVCM_Pz$!F752
zBKvpMq<Rl~rOP=Da0QZ}`a@EI9QWNb#e9UC3Uxz+UaoRPNK?`*&Tz=&K_wO7AnVc$
z{Z1uvl|nYQ57~XoOC-`UDvU%#K)xq)68jX9n}$5C!>~t2)!Tz4a#V8nfW8EGQtyw%
z%6j&I8eMI-xVp&gv?QZRSC93lxnz80qKfA7{%bl@VPZ~*X$WnV6p;u@hOPIL3>dmc
zEar6FJ`>M`riimD(1ULzPb@|Ji7O1PjzBir#Yc1o#k_u$=aD=UK27gN!$|d*tMT;n
zfw?SrK%X3WGuq=4L+(vsX~|a_#w{egQl7g$yvSClU|e~vp?5}Jr8j|EYxtw|pB}gc
z&GGn6_dfJUL&w&$KU<lQN%Hqho_xEWvx*)DOZPm2eP%;_w8AEg_u{4&Utkgg*D^yR
zt8b}o_`1sF4ALN`((=Pvuy-hT;{LJq2LOR1r<q+PbE&H|yL?rekOb7-6#-Sb6Lv0#
z_T^;)9}L{qMtb5g-EQyrdAi%_6nS?w(@YSxp^LPQ;I3?_?OFT1_3oj4CV$(-RA#_{
z(IfpKKL}z~98y%xZk!a|kS-ld$%@s%O{-n9&nB%Cp+vfq?sBm&|CagK0^z8kA|3u;
zby}=P`CNZ4+Q4)4DAb4qcG2bUdqBJ;@1r>DKb|h7WQJkzZqt%)h7;PraP{ubwKk%h
zP`cqw7&1C<uGh#E@8z9U<CFGkV-Gs6ydvPvA4#ZBT#YX6M3Q2!8ev>8&_ZaXmQFsW
zaQb4<3q@<p9b`n!$RfEeIC#P&su2eIE(I+gMakccsmN6!5xV{+=f34TrDh!t;#)*9
za%YISC}jAA4K1_uAji-!6~MPU@kU3yZ^LkEWsU4Kw_ca~6&#4pXdC}(b7{Ya`vZZl
z#k0kgd#+-Mym1GwF@bpO)c&27R64T9YpACU&sgC6U)()^npvaQD#2mqvTu0~QOqE-
zXAzxoP`{m0Sq1_m+$)01-pjGf5rtWysVsIS72j0B)yqqEdQ)d<ViE5Qp-o3N2Wpv+
z_?E*^J53i`gw!a#?NZe%eFy!+0XuTDh4F+CMPf*b4Yvyq1;<d`$2z#cQ6IRO#Eu<O
z=c3z(jt+_t;!DP{Yji;`Lr{`9vT&H5u9S@ZX#1g5L<(Ge>VXDDmS%FgLG;JUpGMMo
z`^|$xM4M$n4RKiT4V*0qAXT(-L)d;P3sR((<?CUlK1JoQRdGSPBK}gMY3ay3`0pRL
z{JeXP3j^z2qB6gQu`V*Urpn@EgliSK9T@j+K>k0WDwm%jC8`-%79>P9M=crc!AYZV
zDK<N@cjCq6WweYYSykw7zS}6<+LEn0z07`)iXuT_-ZWP-z}Y!hPD8~clYeT8<VPzq
ze0BW$VWs0Lg8YmVBO9qM4Anv3y@Q{2+3yu7G*#I>vD60KnC9!^9|gc$_ZIMX0;}VY
zpwN@MzIAhU-{Y|nG&FZqTz0<4tcE{8zB~T?REZ@@K&2`Bq)O@~_u`|ckziq=dv6xy
zCO1uTHE@fY<SsR^=<r0Zf{;c7HBPlk5`2UaA-LQ))Gu3J&m`&5D5o)QmK7pr_Y$UO
zWN%=pMyOGTo#ROs(HjCE^TJ`LqkeF5V=C`Hfz6xNx4O+u6`j?CUdPLF#N-?+y3uSf
zchuSW#XuFnob2WpWE!m{63L?Jtp}ajq56bRIP*F5-<rN4$Er-wK5jbbFLZkRh31mW
zVuZOv<WnuPYl54qTHOMOp63yT`)qOpyzRkiL#u?d8uAqq<lKn%Gi^>;`Zv?_8Z>2@
z>Zg~k%RIvR<uUna#m$69k=GM*J)y)QTRKW8#FL^lBRi0>WlpKP^(Mpu^N1<8`cu%t
z_uIf{0`=?;H|y4$6Ydhh7Sa$H1b%4lJ~I%X2{L|%lu-jw$NzkdSh4keaFm>x1g^(Y
z28UZK;<3_`+Dcq3GJi!mX0QM$F!8t#0aTLY|9R}ACK52|y}Z?HfGOG0hzxvtaBIVB
zb3b)f5_H?aM4B5w`MLT`myQnKkztQLT<c_oE`!s2yPtsK1+A00XRn=bG{M+TUMZ%6
ze0jCiiVoBeZ5!2Y*jv8fFsWs}amw7v83_ApuMU*X*>Ep%dJbYy>Nid+GLF-f=#`i)
zKQLwDF8Kv?wS~E!yI1*pQDBP<p|qzA9wq19*}eOMTR)ncki@ReWBEG>NZn}i-K^YY
z1lb<dS-?(^;@QXL$Ejx6QF>m$vO>)9iTjWl8&IMlU}mtjik78Tmz1+&$0?*DcUW=N
z?J{bqmHRW!ZXW96F7U>L%Nf2k01`HhKbuK_df_)3-lNh}Uw+1miznC37)cEU&14pZ
zzXv7<Mt+M(C~#{y&xRvuyASVf`JQ<XjerSbR%!ykX9VY<0*?+`)aE!T$}+kmQ;>i?
z3QQnDyWNkcayEA&v=zX;Z&4mnh%#7H3t4{$rIOTJ96o1RI@gQWz?Rx~O2Bt23a`R1
zUi0C{sZJT3Fro*%jZIP4x72aPW579t((B2w$YR%1%|*K@0LejY(r1kv|KGz{<A3P$
ztA!SXp9isVIS-<lG-nj+sP!Xa`%o53qKJAa-GEC5t>2!EruTtTst>XfJ4AReQ3rfX
zxs|x2SETCNdJ5L6eR|<U0?!f7Xt#R=IAh;CEht8*_-_U9TG2nnv{Ad}$iP1Og9B?@
zPO#&a-dcM2_Zv>-WyP9^nuY<dbWc$Dd$Q$wpdg)caaGN3Lf)8z#eEQMP$}41y{TX8
zUl_8HgJH+LdDdNnK^ddnpp|q)wx&SA3t-O99k^R?hYujiUB{r)N;%)}Y6~BFA>=kd
zi{<9IcH1U}m6*xAm~`v+-bbn6cVAy!t%5(-ITM32GhE+#wS?=)<(`+{gs@$kPEWL6
z%{Fdo_ciL^F-DhFUp}|)Q9^=Y58%641WD5cS&;!OxgE|gYR6b~8c?AvxS#a&dbs8m
z1x<&jmnwEjIQ@J}g)O8SDw<+<@_f+antk^cqd;=+H}bA~L^;M7{Hm1E$9))o&(t`Q
z*4<|1|2f#>PGF>coV1Am%aD*)pR6P!Z+x|Iom)+NMl<08zSpb4xVu+#(vOA*2J`~b
zzs+UE&Q-9*_`xvBp=m;-^j(O6OglzKTA#}h@*7mM>+VwD{|kG{yYS;=qlnI>Edf0>
zJvtaz*xa~eyRM@k>Qr}c)u$%R{SdR*jo^Ox)w=!hg7zGg6xT`692?nup5yf;=)YJk
zHfV0(b<<!rIlvnJ#FjGQQJRVU7hEu}&~7z>+BL*~htk!4GE$S!7p%xZOc?1g;S-cy
z2l~@>+q%O;)FVt?=2b5krox9hX_!S<+oQ?-VW8H+{Vx7i6vx~j5V)|=jz5OAFXe;t
zEpl~&fbWzW9b<CU2<hkAZ;w(%*e<2PFyPd_02eu$1b`_w{{w(~0ol)oI`{BlNTo8b
z$_STJ5Lr~3J#&jrSdfbNK(4ZtEqgylO+_F<A2I>#D<)eW5l1dc;$q(~AS@7fURe)b
zrH%2kk+%`xIQI#}PXq1MJ<Op@-VVCVkNG^%Dd$l<>E3z(T9dQL+evhl+~%WYARY;T
z=Z1_g^8z;G?%4K71Bh9s&_3b<>ITH?S{#n4@=<NUorN50epI&<g!fxe+HHx-n2^@2
zr$M$OZ=6q|LoQu{=os`;nrlLvB$DVTH!*{R0%lH~dVeVWKY#_o7!$<nI@vV@@^vwJ
z3pTz-?aZ0jDM4b#)N-EkodmlEQZJcM;!cy?w#EmIyjcTu(^eU+u*8kq9R_4mr+i4!
z(ZNg>xl_>#u53)rB+HBxORk;|k6d-3|MKBUm!EQCdSAg>OQ)<0#Q3F4I*XC<U@_&-
zO$8F{9gjQig+RiII@a=&Q>$oE3(x6crSe2Zmn1&;-_d5hqozsMKe}Wy!o(lmNM`4V
zR@;^cA|7Z)A^VVSmSM@DR2z)7Q+Uo}PR<Z{j;82^yU!WwybrxLv#YwW^p2CYsiYi2
zZBx>`faVKn$=C)Y;clEi&!4Q){njp}Vs7F8{T*WjZchkQ(D0mwzFPjr6uv#%L`i)8
zTO&sG_4h3D587$+=brN+5ZzxgOP?8Ml=%5~<j@8HDE2|^l3^4OPBv3BuG$(-Y|l(h
z4R85MS?Cd6*j$K!w@t^t_(-H|r|gr^r~jnR@bq;}qLFURfJzQiZpr3iWPo8d(bL{-
zS}QLIon+R5gEs3s_O|VOYb_U)wjXx;<q5BP`PBDO^Ndu3FyLB~Lx6XMdt}t=wtUc?
zSTEklpllh%+>(*7L^5w}Izu_CMOYpi4wD^F<27l=(%cisVB6tNXX5&F+bk0}pv0>{
zzjy<TXpcUKRWrxztjbH@-_@Goxetfm35h>^HlQ5b%*49)k7YhxGjQj1y658dCum4w
zs=O(+Uf{g}oYceawITK=isBc=foz9W;Y`As&EjpLd?^t=X>1Gz3N?$Om@q^8O#Ay@
z*tKrQ0TVY;)c;W!o7@6J@e@z>%?euVQ3^idte+8Gf*=l3OlBZ}{Zm|kNz+ZpJks2O
zihOFoOrUuy10$u==&NkKInk%R#VTCx`T-UbNCMwYX)sO2p1`zb<*?knrx&c@33Q<X
z0`mW!h1YOS2dZ*)067;5(NQBopAK70IBUq!V-WmelJXh9@hW(%)bF;SP#(j%V1(T_
zkY#HNs;BUKY$B1A(%FJRS}%oHa}+bWo`hJq@lxEs)~#@8!~_>rsbk^t1}+fmwYZhL
zqkJA!>4HuVF9^$y*4#K1190+aWi_Cre?a^bN~Sta;_%MGhqk`daU0kAzpit|5s4_R
z$8r>&M}Slh7`Fe@+5v71fDa>P`awlD^3BK0DGZ?dKJ|(=z_{<Xf*BIVmX!x7eN<Am
z7eJ|{(^z>s&vd1yP})x()ZfK{+>~<j_Y@%C-HJI7y^&$*?*!<&$+~s_X~O%fD%T2Y
z^fLyB!_xpHw8c-43^7lcR=Ql=j2V^ejD^YN=x<EZe=9lZN`23}L&8kddIT_T1-x{E
zTeuG-uBEh!mRwvBl&x%th-kRxO0sP58nz<PJRiQ?bQI>->$njoE@=om1HNeo9m23|
z<;(w)0qhm?@<Io3fPlo5-<FrfFJPe1f_1Ro;`8`Oa>EGFB(?c)QwZ{5)w{=A<h0K5
zs{5|8dS3XLu(d<Fk%%TUV#FoB;nIYvZ*>QS_Lwr+m7xH!RG&SsDGg2m6LGcS^_$kD
zUENy_7xQT)Ms?P21G}i~ktY%df3cQhdS=cO>XAN-8uaU6rkSmcg53fAR6wz0l=1Z?
zyk~1Q`4$+E^VOg_#hs=sxPQlFiZ&l)Kl9OW@Aa}nxjCv5Vk5x~^qzzdxG#x}q~z(6
z8FWYWY$GesRlygTVL)7d!>GPi2esiM9$dpWL6C}<R>b0-P{+D3^kV$l$)dwtq0g^s
z;!cMbcY#p}XTOF|Yv(Bjc^sIfaj<l?;r8IK)PoRH4vHRs6*A8sCZ1Ml_MiNgAZ2L}
zIgQ+oRvTM(&S|C;6x#ZYu=$Q%)~ExcT^VM_j1wWtD#tWuj^}35k;UL{j7Zet#Dv@(
z?U0>|-W}yFWYo2tF&&i3m(H?VZz@i4o{&zs<1N{3ua`-93F6ZtsB2`1N(W(V-0iQb
zW2&I+QDq_A#u*?3OG}l}25Cx`<NAQ$#f{C3^pp)!ddFckWA?U&(%v%ZSi^s6rXv(S
z3z~t6LfyIXPO1&9x{gX^=^ET|@cIv2J?Ym+;qvbW0fx6J?_?n7>yJ*8h<5``HgG~x
z2Bm4G2=Y!VB1FPaiL1ZB6TU=Wcq#uCUlh5q=RfQBSjWsm%tk<KW>6_R9JRuR@oeOi
zdCenHJjXK!>tz_=8H0DD_!D!>XDS?PX|xim{m8sR?lpMfOypYtONCuxY|%k?jJIGf
zeNVUC1%PB4q#4Gk$&Z)zC9pkkV#PKVVQ#p~miNm4Ii}p+m5%K}2$eR_7T*%xrntzv
z2=Rd`0BDHqeB@`Y+Qg^6Ss&A*f_A)V-|+ubdQiWu9CE?EH|PjhfKoBxi+H7^ZCvk1
zdyjTAJH9v9;((J5Y_k8%g3>H$kIyf4{ISXcMHkAml+$>MDm*~y{9_Vd%oyu0J?t}!
z#IVyA;;!XA1>RTz*QmYFyuD$M?bg*Ew9#Pg=|@(A-)mBd9$DG5OD1%rVIj%pGId-h
zc}|BSC0zb)k!}|S5R94B3H#x@+~l*A8smeK__=NZw`$rqe+HyaZH0BDdhH{{`o6lf
zaBglpXC${L{&+Plp(W3Oru2DWj1~X1tn7fh&w}QWYCl;mF;^6d+%<YCn0J$#`lX@k
zCOmudKW4qJDxeb6P(3YQbmYtsKS|5{!M1Y$p>}-hcRXO1doK#wz5~fCV&j5DWY=rd
zvTJA_er>`5rit^H&u9{7fvc|RLXGJsJOcI#r@vfdwJY3Kf*PL?H_gq=zV2;SWFP<z
z679LjFY&T#yFocz#!vfiYz8S0FwwHsc*K!JSQ1-FO!z3z;Au~#_h?($P~ViO?_itq
zg(gb!OCGNmr)ws-1VDTcE)(?ua7@$fQ<`yK_2l2wOvQZ)!wd_Zr&?2`D0aq2YgB`3
zkhA;cISy7Ci^H_$ys$v)?nw{cfM7>&7S!K4KX6YL?yss;a>V4y&1;>j#BrOl!3PgJ
z{HBz?p;~b^&(5yqj3K%^^3B4aC*H5{Dh@hyw|BHpxlCLIEho^HA%A78B@{Ml^(2bo
zYg<D%-ZwU2y@v>>Lp$f6I`SWg<glfJjlbJ&dYnq;9`ZV0dhSf#LgL91g^lh0hN$63
zTF63&R+VJE+kxM%|B7DEpmHd%e0__+e0@;tdYCofB&#@FlzuQUzI>DW)Qbc(1e%jn
z$EXJTmRWu%5@0Vwy(diM&esHR_qQt+3^jwgjfvp^00lb)EJQuf^YOK(2%qe9Oyu(0
z-(SY0`<tt-B~*u7M{4Kz@UyoY64L}wfrSPh+Ww<dgjEU+W|K+=d846T(ik8UAPo3%
zz9W8S!qN{RYj2GmyGzh+t5b8AF9^m5P@uV+S<36qL@<};*t3dQdz-UQzr-!d5)753
zzOpfLm?vMo?c=k%e*(Ta^7&GMX2ntjg%--EdWn|n0$}N(cMfU}-(&XL0jz*ele_=(
zR}{B?3JR?9h4eGZycxUvddgw)ML5>YA`aaHkJ4ppOkzP-1N461zoS88dg?}env~=t
z9hy_imh<yAp<_+4qkm6xd7v1CxbXpv{qHE-rSA}XRtcL$nYk{Byw(9xbB#3iOdoj;
zQb!D$!)0xLsnHN}Z<&QKE-@<l=Qttt9k>QXKFMZR->hhlN_QOq!&7sxbdpjEvFV#K
zNp04t3wwFpdNVTD?U^jzzpRpk@CvUHrkU^^0#wWLtwlxdYM-1Ird&MD;XX8F*=DkU
zti(OvW7m(B^zZYLX*GDfB@dOc{Oh*724O#ra;q_#m`Z3Jdtq96Wzo|R>~HdWNO8rr
zICmcU=MEcGDM4fD3UDe?N2AlGj>2@R7V^U4s*JK?juJhVbFuhaNB41Z5=XFV)&C3!
z+r|?t+X4NdDl9fJ6D<=ra=ByP&bNZidhN;VPXtSo>txUv6*=@ds$dhDP@7Ipw)?J^
zel1qt`ATE7QdX6W=1<dq7XgXq`@lweuO@gQ@SWeb`{f@43#e?RJv8=qBhLRLd5;Fh
z2F3x9EeNhYTg}LSGBnxCmyk`<=~A)S!i&azwvrtI`@RuJwi;r54dQQQwE+!qOZ@?=
zf?BtdT|#>a(&qar_w4F}=Kzy{P|<Jr#Gy^FkTn;ULl1rsO9#qPm>yPiNvCwU(y_DW
z|3_+`*n$JX6(k3PWR@3S@ubg|hA%!~rBPuP6H7_>EqDW2D13H_`~B7`QJ6+tK30|D
z>vezHW$7u`)Y<@O9x)LzB|oQJ>ob{yN%QgEw*cX!J#d;?4e(}Yd)3jdO@QVYQdTYx
zTBKWHN-!8-cS0Dui8tsZqFRJvP4eB^4?Pw<qD8z$jww?I*JmqbVXSpbrOrg3+({v+
zI(a$?>hxWjsF!4YwEB3CgQvJFxImAdZhcWZ;fX9)L6u|6(2N<;&a5Kss?mkTgLr3P
zm>La4C|_^L-`EO5V8ugMp5>2XC%3%q<WSNY{LzRHtKX6$*M_87wSnNklv;gBC0MwZ
zj8r~`>yV@Wskb05|8=rI>iKWhI2VCxDcIP6jTbXheNOTe!)`|Q^jVrmw`on|4cu9N
zeV%fikZ5_N1sbXFU+2n$I7?ju@T>Uod@3D$pT^XZW79>Dq4ilJIO}k{klYjP+{$$5
z2-umRk7wBW4Cf6pVn7C<;vFDnw562D-wU0w(9<2u!9Ft(N&?EsOGebjooM2h1*=c^
z93evsf~h-&U(EgyxmMBZ#Li{w7A(SeP4E^$AU1&F96jnXTn$pm`kuNTaMN9^ZKXgd
zza?>1mnNVZ$>XEcZ<@~lD1Xl;7Y`Lz0R?by)+?tqaFU|0KEm~UmN*rRIfgS|btU!I
zva1SxvCW^dNEl|I(9xcgP>ZikpZ3WLJVr(`GGE!<nW^};kFQz&giQ49G=v?M4s1?2
zD$uC|u<^xz;I<5pAS@YP9`k8dWgpb#8G3{4MnlYxe1%&FwnxX%Y&VJq8WnO-tu?ed
zU_qg@!eeKI#B^hwTkSl;AJsje>=;&)c^QSSAfjBTpg&OD)n%@We#ZVt8)*+*n@qL5
z2kR<#+^te+o2=eEc;E;Au_lSUtx<H2ix_9Qfu(0r98#84BgLb8$Mx(j0X#~D1sSBD
zUT#~K48h~cLw4&}Ixf79Zt^Li&+r*PUOBr_olK^EnH|O-^Bw-9Zm2{U6REw^0eK<r
zs#j17!6<}V@TQA891y+drO3yHvFZ2FR0gf|tmUJgGQjjAsY5ESms6#hl573i&vBD7
zH&VQC(38vd#A>U^Qe(_I0qwHt7LE?{oLcs}^ADO);O*YC+#qQXZ-gLU$`Q=}{2G&T
zUKBzi`ZpM$4syN=v%0f7(9YMcsTgPY(0A?el>e@PRc*@INVyIoIlTTG?m3E3ox_?T
z@{-PI@bYj&zm8{F;z6n_;sF&UBztGcMhk^P@Kom94Ri#uekZ_oFzL|Lw8c5OQDd<m
zxmGimZ1thCmU;kewsOb7j?3i{tV9QI_2zzQm@|Z+<n)%Qnb-nHk;<(vQEIbM;F+#L
zdoO<CHQdxAc~Mk$+^$z+`)}R5WAq$aLM+i+fS1uUf3v<xU)|(x5!qVw^qCeaqI`Gz
z#N9`-4hHe;8RbUaQHs|64jpVu)T<d;ASDvyiCo!MFn>8;eKU=}PZ`Aswv!sBKxtAL
zQaoS18^MRr8~IUugLPQ&SlsF86B2e@VSu=YpO783A{MjvV3p)gSKIak4C+)bFR5sP
znTFW-b`+2iKOfquPJtvGH;SPrK0R!Q2u+N07FKA2y#}WQYN%*X>hcArfsnL$!*L7x
zugVHGFEp*wWK{q<d*RF&`{2s>1MM(>1-$RCNu5`W!=EUv!d^P`-t8r08k;EXla&ES
zvPI`~h~B4=?e`p8_RWz2K9L!BUbwmL?|gb{91se^(OmJmL5+gVTBPR$dE#Q@FVk|c
zjm;x0-#YQ2Ur!Y`b^BdB(c%B`|Eq=PnKg%DiI5-S3HLC4*kU8{tD;&mH-2nGnPUwf
zuFovv)-&%5CyVEN%Nr58Sf$vgG2UJ^$uHQ7d^fM{_hH&S9ExS+N7&q)`YGxkVF^`t
z>vXF|v^qpII|)v*_PFYXEY|s7rW88l#J+%VA0M2<p3ZC@Wt(>t!?XToj=%b&s~S6s
zIW5gT-W&o=c2MTGY=Dx{ojGgn(ac#FaP%UtqJEidL(Ek{S6~bK2+WV&kKYk;9E20%
z-H{fiN5mi6L2wYn^s;z$03jo?!wd=r_qL;8F7TP7Mbe8RvqjAKNsOrvqOPmBsPk2E
z*pC2%QJ%#t4Sr8;R?0AGPHm#UZj>}nA=w>TfK6f(jGH=N3uSrKw?e=v-^^~X6JM(%
z3;E-Zx2_w=w5D&Yqb#IQ=(|WM8Xi_$=*!PSJ$ENxMt5$ptF@7HB6Xu!=U#Rzrl8-6
z#BJk%tU-Kc-mp6=#I?a<(jO}wM-E^+=f~A(d>cQByuS-n51ZPATmOvPG`I=cc1}b<
zD(@3Gl6GauRtDv8`9^oPy(SLjf^WQGIsM|dri5JON)Gqe*W{wh@ufKW3PF4_l~9hw
ze4o;(_1s4c9Nrr@|F@l-!}}!wBj`5Qt9Tjiu&g}@lcfpziWxGt_o03k+24AB8fHL5
zV3+(q*w(8{Ldg~wl`8Njwz{x@kARA`3o%l2h9R0f45c~L^9@q_WC=Dz>rsf>(g&#j
z4Y=Q*4Ywbmx;pJ$gW8q#!tNSWIae}wf+$_;=;p18^%tJ1x1yz9nPecp7|0eJ0)Ec)
zVHz$ivZ(BbD1=ZQZ>b1q(voDdNQC!B3yNd#`HxViN=kU!@Pd!VY#t;D_O>pN*`9F|
z0_gtWzPW|r64d*gonm|t6U2LIEvsWXZmcqQ=~psOup`!s8$X=?+(WJg0(r@q_+=<`
z`-$T$;UiceXoDi^T1Z!G2EXJfE`%Me1UwB<V+{9e$T<1;6^!UGrX=E5M2GZ|%j-R@
ztzYk<gqas#t+D9=LL3o=mmgMgemBiu|A{=&!bJ36)Ew7%B*27!(xU5S5?@h#%g4}l
zC4cT@hn_j-BbmeBF<ViN<(%<8(af_wM3n;vuum;K38fs6|KF%Ad8X_LCX*}UDx2MV
zdwY}zOoqe98P`%8>VM^s_YJ97KZb1}Iz)77mZ4&d?a)U7_v0l60!&mQ$iWO}_~Vkm
zKQC_L>n@`lH-Deo_mS0xeTEX+qvQwE*6`KOPuZ2VSt(e^cv$3Jags$-YMn0D6t;X}
z{aDgEM*fXQ>nLko1YOFuw%WrN8jRHDKtzalW2&Un9NLhFurI@O3Ogo<A1`JOQ)nls
z^La)NgMY1yQ2?H0c+38dSD$9O$~4Uj*`s|_KN>-~!89O-@_#gj*im8{92CZmt_WOO
zC7UQ!=C`v9;x!$oc-OZd;XZVz|G&o{@r?Q}Q)`nkBO>#RLup#-$`LO+S=Rk!W~9Y1
zRt;Fs#o*}ucG#EXrl%ZGfIAUw$$2qnokPr+jNVA@wg6~+tMyPzXlBH!m8Kw<wN-xO
zNYUmZ0vpcsv?{rtJIg}-<`1HT@S&xND-g<v68>z4GNuZh;$-p@fOE8_=d)!S_s7fi
z^v3#uB(1Xr{XU%iWNscZpB8Dju0)H!58)>CC%)UG!WqFNAAz3PR1o`MhjEuF&X|<*
z1u8=^jbDw(^1>f99D`f$2^a4D_M*@p5{LI6b3@RF5Y^IAsij}Em`NpTw-RLgzL->)
zst(Ew(>0vPoP&d<oUi>As?5z5mqOhr333K8bHc?gQ`^^yE0-^tqM!7wu`6(C!n7CA
zsZeGv_kRL>Z{GjGC}!$xLc$UbD*67735GX5L<%zy)P=A}%x2k?Atw&W%+0V1_NRRm
zu`?sU#!0wL*}v+I8RJe)DzaH|FBm^cRJ~XPr<s#rD}+JH_6yaH)=|wLKBS<2NYdb=
z&4U23q|b#XkqGy3LPuWi)L_Vjj|Py?WAnsmiT%%lu)AxAfF+Nkr?WtL)2JuiYt>Sl
zag9u9R{<Qoxm>U3U^CQ`Yr8&$k0#bPl4f|sgV;cl(%D7Pa@%i?_Gr`QPnN}*Q9$tk
zRloN_5IMo%J9Mpj^?Ub|1p~@v1AHPjQTQrQyXBx(=Qs7ey_7$*U3eln772h03Myu#
z*Vm?kE7^_2G36gV((A2r_)-l`=Dg!^2QAyiR|y4wK50A1sneBfBR#x7fD%1y-_k6s
z^h^BdJo2LRFgjAT0WTSaR{H%rsoppb#sdw0?9oru5^rOIi~Ts<A#YBgZ+L7{Ovhh&
z`dNbDCdDiKQ?ZGIZ-E1ca&(BXZsCtYR>@gpuTl8c1T_T^@}Um&y+2kI{0Rv$^GgUX
z<#C5N<GhI0Z_C)zXbRj|iNWDY2~{&w0qy*M@(e43mmoyjte4iT-!4t@0`V2-c!owI
zDV6pib<4K>y?%aRCAwEfP=oqh=qc|XT3=KgY)`pvjK8^ZP!gx?jr-3mgA4ODFgHx*
zi(jz=HWovd6)U=}@S51Qf>SH5Er&1(cDRz?X+_RFdXJm*_jEcpn+{GY>_(kMJI_1?
z%_MYv8pbgp#)CMI%E|c3W|`Ae;ZuT6YB9uLWHjZ2?S)L|5Ox~MSk>kHv{vIJ2RDs+
ztE@x4V!2xGPxE3u`gJy)XPX42>#FlF601KKaAus4!sYFU1YhEtW!XH>(6-R%#9zZ4
zI8n`co&>bIi-CBZ&MXKJlQWuYR8Dqr1;l~TUoY(|MNCO`RLRo9LdO?3o@}Ogcec8{
zj6Hko!`GTZNbX3Sf@GKswS@g9zq+eTy4xR6idgDH6hCq5-z#CDyWN6LH@RrFlxzGu
zty`EJdUSDBC;7n%lqvhBhhS4emwIsp=(@9o=<d^t32>FuK&KP$DvVn}33!9s36krs
z@^)-Lojd?0biIn-HqcQ2yy2G0RX=**7eq!%nTY}s$%=|sw7r}ddacTD^6-)FM2s8|
z{N3b>37WjroWt*;5u@_Q*8E%+jHypu_i@N{T^&U1BDCME&`iUOivT+=vzj%CwU+k5
zTT#Xuqa?qIZ-;f?h3g)+f@I0vguk3cnG+jT4S)u-z8W1qL#CcGfq30dzfJ6|Eadtt
zs83cZ>GJeUv60eq56K4~A9-oHdm;J=uAaDG_(a~B`!BU(5P_hP;C}8lE%><h%rGm!
zKU2Ys2-2P-y63qW>FZsvvsFq1=e=1gVn9<4?WXUq6CFqGLL;FSuZ}mv0ud4w2Pw3X
zwBEY&fJEUJM|&gaoN3~aLT0i~=dSb%A*(b4+uDYM-6lcO0i>uSr$Vp|)FAMkg-liO
zMFqQG+YbhaE!Nft*FqRdGZ&_PS3?bDJh+tUBO{yJItuZ@N}G$7i|vWN=W$;brQV{u
zT6%>R<inD;+Jj~L$cTGsZtceO79h})Y}V*SnoofW>t@oteB}-9?!Lzy{?(AppA^;0
zIfiT(+>jj(A;TlUbSUHF^gQVXtyfof%S^HkUmaTs29Pq=wFuX$m&`m$rOE&nCZMGv
zbUcYSy6Hb9O^#WX-~b1C39UVojPgU@cDr#+*o#SijMxc9LB`Q37*&GPRfBoYD-QP$
zS2#Cq+2W|}f7;Xmdrc;T?A9jH;QO@LnS=94W;_yfuwccJLzW{Z2fN|<w2Q!nR}C9Y
zlOeeDbN{oiTiuVbO=pQZ^CC-n1bl4qH^8Akc&-|(WHd2?wRmHRM#ABhQc$r@LyU~Y
zMabNpp96HebV0FfA1XLktaTRr0S+pGwTi_9PF?<K7dQR<oGRXtilH(8?GTu*Ifv-N
zA&Mp+)Zq=U$BA+;BgqAsyg%mXZ0Oyn$_XCxBI6j3mwcCr?wa;LgIr=N6K5w?g!+WT
z0a%&RceGe7ZflwJ^j)E|xlLObr4$%ivuoQ@t1zrWm-T5*JMdwM%pO%0g4_3W5er^}
zX{=~a9QmQX4ZrS~+t4}#hwrV%rr<U<`Bp^ygGrKm^V+^b!SolT89+JKp;dZfL=5E*
z<^05By#HrPg@zZ9obYFla>46um}|o@sArW%Y8fdwABP(fd^(!1R|=}_tWp9`&ov0*
z69>lkeCk%^LjAB_aZWbPY944aTP6Elo(T)Qpuo%XDdwA;bZS5T%OKQzsqpY-AiBQr
z@yw;5DQlm17h>Z$QiSY*3b;zL7f#531i@p?j%kL3XMs>;<;SMZqe?LnN|Dj|k!&~)
zqUTx3UK!|ie_GF%3moM2kQ0F0>w0agF3Z|lP`+Xf&{Hj|q+Z59M^yoC&_n|2?y;Nr
zVC#dvII~o@c8J^L9Y@yY%!n@gF=jg13^F~E!GdGCd5KDUckgm?;QzK%A|G}~X^1j;
zg#qV>oNg3Q1hrr#(EEJry7u9qGw?fAo~$63919nRcu-4iRbuQKThR^SY=iYIzG1hx
z_NlJoOvd+cbF~Xj)^Rx|dgp)Lu_)Vxpd!BcZM?QC<mgWbBq30&$}BwM5WPGSzKny{
znNg<(!x#I4_UH?9E1$R+3X3ucrq*h)6)!#&j@@Z2`sb{3zt{Pa#2F&IygkC-s80fS
zyd``}R^!>6XuuINNnxD;5p^r{sr~k(i{_jQAB6lx`(&^>{#%jPsNyVJ3q)FDIe7*>
zF-7EJ7h^l|;^)rOX0qF;AsiPCvnM)DDO_x;Jy%VVuWD(_v5qSX5nOpb7W?T?JJ!YB
zH##gu(vc-PnD_x@+#?xzCzFsFh8+FpO8Al|Vei}{6&;*Ch7su0wZ4^GR9{!}?!XtT
ztTAb7gRM5CGzaI_kBA6zbq(Ye!74HBw?%~+!mi#eZ<H--;7BX|goYY+9%5N*FdH(k
zcow_{*fcz=CG`xXVK3RKDq$RsffV8?1!Q{sBIB&@tsNw*$mUQ(<H>C%s`JED$UW&k
z<0J@Ru~^kTS!hp0)b22vaK@_(<E@AG9?HOLF}?49*PIv>3wX#4_P|=ix;gjz98%kJ
zJaeke?;h;4z6Yn`z}k0~t}fM4iHBEe$2f=mSeKqL`{OA_Ma05_Fxe59Hxw8FaT3^;
z?W$diayc-sanMkejCVjStaRVG`_}}Ihq8DW$L#$V<XW0=Go~H1va6zfy>g!UR5j0-
zJ-1+3jg*lFjTtApuR=cSHKOekZWyggmMUY`ry3E^Yf$c&K{_G#Bo%&DRQza{4++di
z^JR>F<^{c{XQXl4lUmf3mX_&m9=_e=R*6vobMJKc{vPoT{98CWulBwbdXjv)5RTaG
z&=4p+sZ3sI-rKBp$|;o3-@=#x`9-`C$=@wP{xj?LiXKp>yD=Znmn$<kd*u3hgVXdN
zF54JtIgXvOCzH>(kwa~yDpWHp;G*JSo@M?hmsATt6y=%HoYrL}9@=SBPt|-7Y(`Ua
z9G50643(2rRumyf%KDWQn+%bv4I(7}gT|nyS7|yC#oW1$rli1}5A?lKYmn+MU<+o}
zj}cP0G(l^QL~4Qft`JAFdQFI9wKjIceUy-FE8f5UerYQl{ACcRg8=)q<y2s2K7ZyF
zXx6(?4hZGN${G6JOXSccj2DJhe91*F;n(Sa_u^MXeTO<GZAGx7(B4k-7{F}9t#0d(
zD)ZD713ndaBG9I;{c`#E!ba{6y5*f<gKK|xqk=4WJ9w$=<SzJIGc(h`3hw3E^RwyK
z4D>c>xMJn&2adYzQTNeePoE4|2Le-@B2ydCXFXPZHMLQzS!bUQMybtk&@196UTXhF
zypJ|Mnm;Kq+=f7F+3k@fQC?C!{_sgn^q~sbGYF#ns$DQOFXb@Y2W+rUv}-4)I!-+#
zzvFXPo?}+bJYG4F^CyXKf!{b==Y`N+!-CAoj61fPr^jr}h%tp#?>0aw-UZ+rAM%{3
zs}2g|rdM4Qa}blyyPwGFlWT7cC?kRq-iM)>jw~s&LUfuP#k1*|W8f1WcO{r=y)^MQ
zP{2KUBr|G3@jJ}v5Ny@u6{38mB<molvx5r@S=Fn)rO6)^p$hb7KE~kW2w0D%iIE0U
zD}m;fy(T{asd<TIZA4>1fD65LjJoM~${JE2kLHs2W}MG>lff5OfuC@3f%BB{*Aoo1
z(9T_zM#@~gkr;KHGk{PTI%La$$@k<d=03=<7a}XSP_saK0Alt(Ra+#GD704Di}5)*
znEC$0qzv9#2u4|qsof{cs;zd|h0$H`?1^$OQ<wfhxm!OFvns%yJ^Thh+8xXL4!hg<
zRG$0T|N2bbZrN62gy9wH+cSMbJ6e@`=X9r)K9XbL@W4>e7}n`G>TE4Rld^bT#Q<w`
zOy_?!`d@`Px3k}W_4zIor~3?NdBkRCo&mW&fB`hsOrdT_5%tW=A~e&&|FdNl!2>}g
z3@?!R@Nr+6?6;zV`Zk4=4TFVr5T#!?zN^8v;MO9hkP<s=WPq-k&_N=LCESExl?*CZ
z&QsLQ0i(q0=@?<eSSS=3B0I2*^$VfBM*qe|d4ba{o?i!mE%_J5-8s~!J=XNz_3s+&
z%irw?&36FZ=}4R;oG)Z*D1KddJ<_^_s)nac|F&gq5&}8Z%NABZL~GuZAqcSZ&v^w%
zUW}eGT-O{d454F5kYd7XX%o%<+2mT`ua?@wb-=`{szMV+l30-ru|=Fkp{F8X9yv;_
zX1D>p$-9(bP=Gq=ec77FcVKHp^qx2_5;pY>zrnz}1Eew4<WUKF21jDk+7ygeN!%ka
zKm_}c`(#|eugqs!#T*ror-!LBUDrP)PUeM5cI9-!aL%o*EQkWpU1s^p!!cEsAzT|^
z_&N4psmh1`YbK^(^Y%?MLs=jzX?kR<R#E6<PsU;466;B&?g2Sav_Iiosnn>-@JcM9
zbWM&SJB0x}0)I=WtEjP4$pFBCnSR?UT^A0_lQvHCZ1K+f0q94yoa;c#F>CZ;5)I{*
z++;%@m$z5Uyq7)b>p9%g;cm)6S<XkJmvWLjGToUSH$YmF;=$d<k^9!i*0gB-2dPF2
zj3Qy|3TRYoYf<yu0cAdNd#H5`ORs;LgDyVjXSX8S=3?D%l<=H<7m1w~#2AjQ$8riq
zJw!XO2J9+wXb76S?k^e8ic@OeF6XLCudpMrrKg<%U%6vK(YX5o?YX1tQO$|6iDv0-
ztHdM+Qxmt)Ht=Lc*VkVxUedv*IRYefz%PxhPPH2$4H1uFZe<-K(UJodiH>W#3kvp_
z(-^9BpWP#M9v2$-5*HuLcj2av-%Uz_?Rsg14S@x|V80SvL5<yJQ+%!DSgVLP)AkR>
zGb@IIFsl(ucKF)Pbf~+#Yl|a2fajqGL{{nOKi({&b!EbTHGm&m{-ZA@j<#=|fGt6q
z1;OGoQROr1!KgFj5o8FOqpS2JuFb-IxZi#LTwA)dj34tOtWNw;!})bImQa|0VC&U}
zqv$P6*&KgVw|-{T15W*O5-H^-kG<fm5g&MEuEijcyQzxu;Z$-1B4-p6A{yI3!=r%N
z5W}C<X*_DN8LS`k5yOeCjzHq}N;kKQ?>wc@6}If(Wb_@1U~`hp#=yk+z{}yJhLLX2
zJjAc-+3Bo+^CpZ&!eAXG1#i&SJ*y%3@Bw8}VqwLsq-a53LiDaDuy*YCZAh{=SkUqo
znWhxe_ppdV{Kb84YLDf8V%&*ArKXGq2h_i@%7La3nS+`6LMr-gDaz<sX*%b#^+EPZ
za(*L@2XqeDFk8i`%L{%90xHFD-P9cd9i9jH;Lv9B;rETKi){bFv=lr`4i+$;(=Afi
zw}4#ePDQAYO`yNxW1KW4Vh)^PjUfIVl!S9R@t7wHs>0XOgv((ynhOnvKMf}+EN)H>
zuc{2kAfhR;-||!Ipc5g7IQL1AcZL*MNbpAOAS)an?Ru3kUL6g~B;ug=ZF)5e(k4@6
z3;Z*0+4yp#97plNVoLEm|8oa=_(P5N$lq%W#IH8-k+5v%;yb>#=)*4@3}aQ|%Hll%
zQ`ZufqlbQkck!PGH+bX@WOs5oQGa{Vr6RN<_9+Nfj#FZcNBUkmTD^(p-Hs@@WEqiA
z1^(SIC*qHk`fSP&hm92qRZEc@Z6Sz$2es@al>Ug{r4bqebYn;jglMP-(BZZu2pK^s
zw#S}cWgIs3OCKii1e83bXc%Ol^nu1r*Ym8yO0L{Yu^Yh6%MpE~db-lCru`@SrxGjT
nSU02UAEam9r6(m}J!*kv#I9oSG;Dc71_$NYd1LxTj<o=4qPg?M

literal 0
HcmV?d00001

diff --git a/src/lib-dcrypt/test-crypto.c b/src/lib-dcrypt/test-crypto.c
new file mode 100644
index 00000000000..463e06562fb
--- /dev/null
+++ b/src/lib-dcrypt/test-crypto.c
@@ -0,0 +1,305 @@
+#include "lib.h"
+#include "buffer.h"
+#include "str.h"
+#include "dcrypt.h"
+#include "ostream.h"
+#include "ostream-encrypt.h"
+#include "istream.h"
+#include "istream-decrypt.h"
+#include "iostream-temp.h"
+#include "randgen.h"
+#include "test-common.h"
+#include "hex-binary.h"
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <stdio.h>
+
+
+static
+void test_cipher_test_vectors(void)
+{
+	static struct {
+		const char *key;
+		const char *iv;
+		const char *pt;
+		const char *ct;
+	} vectors[] =
+	{
+		{ "2b7e151628aed2a6abf7158809cf4f3c", "000102030405060708090a0b0c0d0e0f", "6bc1bee22e409f96e93d7e117393172a", "7649abac8119b246cee98e9b12e9197d" },
+		{ "2b7e151628aed2a6abf7158809cf4f3c", "7649ABAC8119B246CEE98E9B12E9197D", "ae2d8a571e03ac9c9eb76fac45af8e51", "5086cb9b507219ee95db113a917678b2" }
+	};
+
+
+	test_begin("test_cipher_test_vectors");
+
+	buffer_t *key,*iv,*pt,*ct,*res_enc,*res_dec;
+
+	key = buffer_create_dynamic(pool_datastack_create(), 16);
+	iv = buffer_create_dynamic(pool_datastack_create(), 16);
+	pt = buffer_create_dynamic(pool_datastack_create(), 16);
+	ct = buffer_create_dynamic(pool_datastack_create(), 16);
+
+	res_enc = buffer_create_dynamic(pool_datastack_create(), 32);
+	res_dec = buffer_create_dynamic(pool_datastack_create(), 32);
+
+	for(size_t i = 0; i < N_ELEMENTS(vectors); i++) {
+		struct dcrypt_context_symmetric *ctx;
+
+		buffer_set_used_size(key, 0);
+		buffer_set_used_size(iv, 0);
+		buffer_set_used_size(pt, 0);
+		buffer_set_used_size(ct, 0);
+		buffer_set_used_size(res_enc, 0);
+		buffer_set_used_size(res_dec, 0);
+
+		hex_to_binary(vectors[i].key, key);
+		hex_to_binary(vectors[i].iv, iv);
+		hex_to_binary(vectors[i].pt, pt);
+		hex_to_binary(vectors[i].ct, ct);
+
+		if (!dcrypt_ctx_sym_create("AES-128-CBC", DCRYPT_MODE_ENCRYPT, &ctx, NULL)) {
+			test_assert_failed("dcrypt_ctx_sym_create", __FILE__, __LINE__-1);
+			continue;
+		}
+
+		dcrypt_ctx_sym_set_padding(ctx, FALSE);
+
+		dcrypt_ctx_sym_set_key(ctx, key->data, key->used);
+		dcrypt_ctx_sym_set_iv(ctx, iv->data, iv->used);
+
+		test_assert_idx(dcrypt_ctx_sym_init(ctx, NULL), i);
+
+		test_assert_idx(dcrypt_ctx_sym_update(ctx, pt->data, pt->used, res_enc, NULL), i);
+		test_assert_idx(dcrypt_ctx_sym_final(ctx, res_enc, NULL), i);
+
+		test_assert_idx(buffer_cmp(ct, res_enc), i);
+
+		dcrypt_ctx_sym_destroy(&ctx);
+
+		if (!dcrypt_ctx_sym_create("AES-128-CBC", DCRYPT_MODE_DECRYPT, &ctx, NULL)) {
+			test_assert_failed("dcrypt_ctx_sym_create", __FILE__, __LINE__-1);
+			continue;
+		}
+
+		dcrypt_ctx_sym_set_padding(ctx, FALSE);
+
+		dcrypt_ctx_sym_set_key(ctx, key->data, key->used);
+		dcrypt_ctx_sym_set_iv(ctx, iv->data, iv->used);
+
+		test_assert_idx(dcrypt_ctx_sym_init(ctx, NULL), i);
+		test_assert_idx(dcrypt_ctx_sym_update(ctx, res_enc->data, res_enc->used, res_dec, NULL), i);
+		test_assert_idx(dcrypt_ctx_sym_final(ctx, res_dec, NULL), i);
+
+		test_assert_idx(buffer_cmp(pt, res_dec), i);
+
+		dcrypt_ctx_sym_destroy(&ctx);
+	}
+
+	test_end();
+}
+
+static
+void test_cipher_aead_test_vectors(void)
+{
+	struct dcrypt_context_symmetric *ctx;
+	const char *error = NULL;
+
+	test_begin("test_cipher_aead_test_vectors");
+
+	if (!dcrypt_ctx_sym_create("aes-128-gcm", DCRYPT_MODE_ENCRYPT, &ctx, &error)) {
+		test_assert_failed("dcrypt_ctx_sym_create", __FILE__, __LINE__-1);
+		return;
+	}
+
+	buffer_t *key, *iv, *aad, *pt, *ct, *tag, *tag_res, *res;
+
+	key = buffer_create_dynamic(pool_datastack_create(), 16);
+	iv = buffer_create_dynamic(pool_datastack_create(), 16);
+	aad = buffer_create_dynamic(pool_datastack_create(), 16);
+	pt = buffer_create_dynamic(pool_datastack_create(), 16);
+	ct = buffer_create_dynamic(pool_datastack_create(), 16);
+	tag = buffer_create_dynamic(pool_datastack_create(), 16);
+	res = buffer_create_dynamic(pool_datastack_create(), 16);
+	tag_res = buffer_create_dynamic(pool_datastack_create(), 16);
+
+	hex_to_binary("feffe9928665731c6d6a8f9467308308", key);
+	hex_to_binary("cafebabefacedbaddecaf888", iv);
+	hex_to_binary("d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b391aafd255", pt);
+	hex_to_binary("42831ec2217774244b7221b784d0d49ce3aa212f2c02a4e035c17e2329aca12e21d514b25466931c7d8f6a5aac84aa051ba30b396a0aac973d58e091473f5985", ct);
+	hex_to_binary("4d5c2af327cd64a62cf35abd2ba6fab4", tag);
+
+	dcrypt_ctx_sym_set_key(ctx, key->data, key->used);
+	dcrypt_ctx_sym_set_iv(ctx, iv->data, iv->used);
+	dcrypt_ctx_sym_set_aad(ctx, aad->data, aad->used);
+	test_assert(dcrypt_ctx_sym_init(ctx, &error));
+	test_assert(dcrypt_ctx_sym_update(ctx, pt->data, pt->used, res, &error));
+	test_assert(dcrypt_ctx_sym_final(ctx, res, &error));
+	dcrypt_ctx_sym_get_tag(ctx, tag_res);
+
+	test_assert(buffer_cmp(ct, res) == TRUE);
+	test_assert(buffer_cmp(tag, tag_res) == TRUE);
+
+	dcrypt_ctx_sym_destroy(&ctx);
+
+	if (!dcrypt_ctx_sym_create("aes-128-gcm", DCRYPT_MODE_DECRYPT, &ctx, &error)) {
+		test_assert_failed("dcrypt_ctx_sym_create", __FILE__, __LINE__-1);
+	} else {
+
+		buffer_set_used_size(res, 0);
+
+		dcrypt_ctx_sym_set_key(ctx, key->data, key->used);
+		dcrypt_ctx_sym_set_iv(ctx, iv->data, iv->used);
+		dcrypt_ctx_sym_set_aad(ctx, aad->data, aad->used);
+		dcrypt_ctx_sym_set_tag(ctx, tag->data, tag->used);
+		test_assert(dcrypt_ctx_sym_init(ctx, &error));
+		test_assert(dcrypt_ctx_sym_update(ctx, ct->data, ct->used, res, &error));
+		test_assert(dcrypt_ctx_sym_final(ctx, res, &error));
+
+		test_assert(buffer_cmp(pt, res) == TRUE);
+
+		dcrypt_ctx_sym_destroy(&ctx);
+	}
+
+	test_end();
+}
+
+static
+void test_hmac_test_vectors(void)
+{
+	buffer_t *pt, *ct, *key, *res;
+	pt = buffer_create_dynamic(pool_datastack_create(), 50);
+	key = buffer_create_dynamic(pool_datastack_create(), 20);
+	ct = buffer_create_dynamic(pool_datastack_create(), 32);
+	res = buffer_create_dynamic(pool_datastack_create(), 32);
+
+	hex_to_binary("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", key);
+	hex_to_binary("dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", pt);
+	hex_to_binary("773ea91e36800e46854db8ebd09181a72959098b3ef8c122d9635514ced565fe", res);
+
+	struct dcrypt_context_hmac *hctx;
+	if (!dcrypt_ctx_hmac_create("sha256", &hctx, NULL)) {
+		test_assert_failed("dcrypt_ctx_hmac_create", __FILE__, __LINE__-1);
+	} else {
+		dcrypt_ctx_hmac_set_key(hctx, key->data, key->used);
+		test_assert(dcrypt_ctx_hmac_init(hctx, NULL));
+		test_assert(dcrypt_ctx_hmac_update(hctx, pt->data, pt->used, NULL));
+		test_assert(dcrypt_ctx_hmac_final(hctx, ct, NULL));
+		test_assert(buffer_cmp(ct, res));
+		dcrypt_ctx_hmac_destroy(&hctx);
+	}
+}
+
+static
+void test_load_v1_key(void)
+{
+	test_begin("test_load_v1_key");
+
+	buffer_t *key_1 = buffer_create_dynamic(pool_datastack_create(), 128);
+
+	struct dcrypt_private_key *pkey, *pkey2;
+	const char *error = NULL;
+
+	test_assert(dcrypt_key_load_private(&pkey, DCRYPT_FORMAT_DOVECOT, "1\t716\t0\t048FD04FD3612B22D32790C592CF21CEF417EFD2EA34AE5F688FA5B51BED29E05A308B68DA78E16E90B47A11E133BD9A208A2894FD01B0BEE865CE339EA3FB17AC\td0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0", NULL, NULL, &error));
+	if (pkey != NULL) {
+		buffer_set_used_size(key_1, 0);
+		/* check that key_id matches */
+		struct dcrypt_public_key *pubkey = NULL;
+		dcrypt_key_convert_private_to_public(pkey, &pubkey, &error);
+		dcrypt_key_store_public(pubkey, DCRYPT_FORMAT_DOVECOT, key_1, NULL);
+		buffer_set_used_size(key_1, 0);
+		dcrypt_key_id_public(pubkey, "sha256", key_1, &error);
+		test_assert(strcmp("792caad4d38c9eb2134a0cbc844eae386116de096a0ccafc98479825fc99b6a1", binary_to_hex(key_1->data, key_1->used)) == 0);
+
+		dcrypt_key_free_public(&pubkey);
+		pkey2 = NULL;
+
+		test_assert(dcrypt_key_load_private(&pkey2, DCRYPT_FORMAT_DOVECOT, "1\t716\t1\t0567e6bf9579813ae967314423b0fceb14bda24749303923de9a9bb9370e0026f995901a57e63113eeb2baf0c940e978d00686cbb52bd5014bc318563375876255\t0300E46DA2125427BE968EB3B649910CDC4C405E5FFDE18D433A97CABFEE28CEEFAE9EE356C792004FFB80981D67E741B8CC036A34235A8D2E1F98D1658CFC963D07EB\td0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0\t7c9a1039ea2e4fed73e81dd3ffc3fa22ea4a28352939adde7bf8ea858b00fa4f", NULL, pkey, &error));
+		if (pkey2 != NULL) {
+			buffer_set_used_size(key_1, 0);
+			/* check that key_id matches */
+			struct dcrypt_public_key *pubkey = NULL;
+			dcrypt_key_convert_private_to_public(pkey2, &pubkey, &error);
+			dcrypt_key_store_public(pubkey, DCRYPT_FORMAT_DOVECOT, key_1, NULL);
+			buffer_set_used_size(key_1, 0);
+			dcrypt_key_id_public_old(pubkey, key_1, &error);
+			test_assert(strcmp("7c9a1039ea2e4fed73e81dd3ffc3fa22ea4a28352939adde7bf8ea858b00fa4f", binary_to_hex(key_1->data, key_1->used)) == 0);
+
+			dcrypt_key_free_public(&pubkey);
+			dcrypt_key_free_private(&pkey2);
+		}
+		dcrypt_key_free_private(&pkey);
+	}
+
+	test_end();
+}
+
+static
+void test_load_v2_key(void)
+{
+	const char *keys[] = {
+		"-----BEGIN PRIVATE KEY-----\n" \
+"MGcCAQAwEwYHKoZIzj0CAQYIKoZIzj0DAQcETTBLAgEBBCC25AkD65uhlZXCAdwN\n" \
+"yLJV2ui8A/CUyqyEMrezvwgMO6EkAyIAAybRUR3MsH0+0PQcDwkrXOJ9aePwzTQV\n" \
+"DN51+n1JCxbI\n" \
+"-----END PRIVATE KEY-----\n",
+		"2\t1.2.840.10045.3.1.7\t0\t0000002100b6e40903eb9ba19595c201dc0dc8b255dae8bc03f094caac8432b7b3bf080c3b\tab13d251976dedab546b67354e7678821740dd534b749c2857f66bf62bbaddfd",
+		"2\t1.2.840.10045.3.1.7\t2\taes-256-ctr\t2b19763d4bbf7754\tsha256\t2048\tc36fa194669a1aec400eae32fbadaa7c58b14f53c464cfbb0a4b61fbe24ab7750637c4025d\tab13d251976dedab546b67354e7678821740dd534b749c2857f66bf62bbaddfd",
+		"2\t1.2.840.10045.3.1.7\t1\taes-256-ctr\t7c7f1d12a7c011de\tsha256\t2048\tf5d1de11d58a81b141cf038012a618623e9d7b18062deeb3a4e35872c62ca0837db8688370\t021abfbc5bc4f6cf49c40b9fc388c4616ea079941675f477ee4557df1919626d35\tab13d251976dedab546b67354e7678821740dd534b749c2857f66bf62bbaddfd\tab13d251976dedab546b67354e7678821740dd534b749c2857f66bf62bbaddfd"
+	};
+
+	test_begin("test_load_v2_key");
+	const char *error = NULL;
+	buffer_t *tmp = buffer_create_dynamic(default_pool, 256);
+
+	struct dcrypt_private_key *priv,*priv2;
+
+	test_assert_idx(dcrypt_key_load_private(&priv2, DCRYPT_FORMAT_PEM, keys[0], NULL, NULL, &error), 0);
+	test_assert_idx(dcrypt_key_store_private(priv2, DCRYPT_FORMAT_PEM, NULL, tmp, NULL, NULL, &error), 0);
+	test_assert_idx(strcmp(str_c(tmp), keys[0])==0, 0);
+	buffer_set_used_size(tmp, 0);
+
+	test_assert_idx(dcrypt_key_load_private(&priv, DCRYPT_FORMAT_DOVECOT, keys[1], NULL, NULL, &error), 1);
+	test_assert_idx(dcrypt_key_store_private(priv, DCRYPT_FORMAT_DOVECOT, NULL, tmp, NULL, NULL, &error), 1);
+	test_assert_idx(strcmp(str_c(tmp), keys[1])==0, 1);
+	buffer_set_used_size(tmp, 0);
+	dcrypt_key_free_private(&priv);
+
+	test_assert_idx(dcrypt_key_load_private(&priv, DCRYPT_FORMAT_DOVECOT, keys[2], "This Is Sparta", NULL, &error), 2);
+	test_assert_idx(dcrypt_key_store_private(priv, DCRYPT_FORMAT_DOVECOT, "aes-256-ctr", tmp, "This Is Sparta", NULL, &error), 2);
+	buffer_set_used_size(tmp, 0);
+	dcrypt_key_free_private(&priv);
+
+	struct dcrypt_public_key *pub = NULL;
+	test_assert_idx(dcrypt_key_convert_private_to_public(priv2, &pub, &error), 3);
+	test_assert_idx(dcrypt_key_load_private(&priv, DCRYPT_FORMAT_DOVECOT, keys[3], NULL, priv2, &error), 3);
+	test_assert_idx(dcrypt_key_store_private(priv, DCRYPT_FORMAT_DOVECOT, "ecdh-aes-256-ctr", tmp, NULL, pub, &error), 3);
+	buffer_set_used_size(tmp, 0);
+	dcrypt_key_free_private(&priv2);
+	dcrypt_key_free_private(&priv);
+	dcrypt_key_free_public(&pub);
+
+	buffer_free(&tmp);
+
+	if (error != NULL) error = NULL;
+
+	test_end();
+}
+
+int main(void) {
+	dcrypt_initialize("openssl", NULL);
+	random_init();
+	static void (*test_functions[])(void) = {
+		test_cipher_test_vectors,
+		test_cipher_aead_test_vectors,
+		test_hmac_test_vectors,
+		test_load_v1_key,
+		test_load_v2_key,
+		NULL
+	};
+
+	int ret;
+
+	ret = test_run(test_functions);
+
+	return ret;
+}
diff --git a/src/lib-dcrypt/test-stream.c b/src/lib-dcrypt/test-stream.c
new file mode 100644
index 00000000000..e4264d5a57e
--- /dev/null
+++ b/src/lib-dcrypt/test-stream.c
@@ -0,0 +1,232 @@
+#include "lib.h"
+#include "buffer.h"
+#include "str.h"
+#include "dcrypt.h"
+#include "ostream.h"
+#include "ostream-encrypt.h"
+#include "istream.h"
+#include "istream-decrypt.h"
+#include "istream-hash.h"
+#include "iostream-temp.h"
+#include "randgen.h"
+#include "hash-method.h"
+#include "test-common.h"
+#include "hex-binary.h"
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <stdio.h>
+
+static const char key_v1_priv[] = "-----BEGIN PRIVATE KEY-----\n" \
+"MIGpAgEAMBAGByqGSM49AgEGBSuBBAAjBIGRMIGOAgEBBEGz2V2VMi/5s+Z+GJh7\n" \
+"4WfqZjZUpqqm+NJWojm6BbrZMY+9ZComlTGVcUZ007acFxV93oMmrfmtRUb5ynrb\n" \
+"MRFskKFGA0QAAwHrAJc8TvyPzspOoz6UH1C1YRmaUVm8tsLu2d0dYtZeOKJUl52J\n" \
+"4o8MKIg+ce4q0mTNFrhj+glKj29ppWti6JGAQA==\n" \
+"-----END PRIVATE KEY-----";
+
+static const char key_v1_pub[] = "-----BEGIN PUBLIC KEY-----\n" \
+"MFgwEAYHKoZIzj0CAQYFK4EEACMDRAADAesAlzxO/I/Oyk6jPpQfULVhGZpRWby2\n" \
+"wu7Z3R1i1l44olSXnYnijwwoiD5x7irSZM0WuGP6CUqPb2mla2LokYBA\n" \
+"-----END PUBLIC KEY-----";
+
+static const char key_v2_priv[] = "-----BEGIN PRIVATE KEY-----\n" \
+"MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgtuQJA+uboZWVwgHc\n" \
+"DciyVdrovAPwlMqshDK3s78IDDuhRANCAAQm0VEdzLB9PtD0HA8JK1zifWnj8M00\n" \
+"FQzedfp9SQsWyA8dzs5/NFR5MTe6Xbh/ndKEs1zZH3vZ4FlNrilZc0st\n" \
+"-----END PRIVATE KEY-----";
+
+static const char key_v2_pub[] = "-----BEGIN PUBLIC KEY-----\n" \
+"MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEJtFRHcywfT7Q9BwPCStc4n1p4/DN\n" \
+"NBUM3nX6fUkLFsgPHc7OfzRUeTE3ul24f53ShLNc2R972eBZTa4pWXNLLQ==\n" \
+"-----END PUBLIC KEY-----";
+
+static const char test_sample_v1_hash[] = "1d7cc2cc1f1983f76241cc42389911e88590ad58cf9d54cafeb5b198d3723dd1";
+static const char test_sample_v2_hash[] = "2e31218656dd34db65b321688bf418dee4ee785e99eb9c21e0d29b4af27a863e";
+
+static struct dcrypt_keypair test_v1_kp;
+static struct dcrypt_keypair test_v2_kp;
+
+static
+void test_static_v1_input(void)
+{
+	ssize_t siz;
+	const struct hash_method *hash = hash_method_lookup("sha256");
+	unsigned char hash_ctx[hash->context_size];
+	unsigned char hash_dgst[hash->digest_size];
+	hash->init(hash_ctx);
+
+	test_begin("test_static_v1_input");
+
+	struct istream *is_1 = i_stream_create_file("sample-v1.bin", IO_BLOCK_SIZE);
+	struct istream *is_2 = i_stream_create_decrypt(is_1, test_v1_kp.priv);
+	i_stream_unref(&is_1);
+	struct istream *is_3 = i_stream_create_hash(is_2, hash, hash_ctx);
+	i_stream_unref(&is_2);
+
+	while((siz = i_stream_read(is_3))>0) { i_stream_skip(is_3, siz); }
+
+	if (is_3->stream_errno != 0)
+		i_debug("error: %s", i_stream_get_error(is_3));
+
+	test_assert(is_3->stream_errno == 0);
+
+	i_stream_unref(&is_3);
+
+	hash->result(hash_ctx, hash_dgst);
+
+	test_assert(strcmp(test_sample_v1_hash, binary_to_hex(hash_dgst, sizeof(hash_dgst))) == 0);
+
+	test_end();
+}
+
+static
+void test_static_v2_input(void)
+{
+	test_begin("test_static_v2_input");
+	ssize_t amt;
+	const struct hash_method *hash = hash_method_lookup("sha256");
+	unsigned char hash_ctx[hash->context_size];
+	unsigned char hash_dgst[hash->digest_size];
+	hash->init(hash_ctx);
+
+	struct istream *is_1 = i_stream_create_file("sample-v2.bin", IO_BLOCK_SIZE);
+	struct istream *is_2 = i_stream_create_decrypt(is_1, test_v2_kp.priv);
+	i_stream_unref(&is_1);
+	struct istream *is_3 = i_stream_create_hash(is_2, hash, hash_ctx);
+	i_stream_unref(&is_2);
+
+	while((amt = i_stream_read(is_3))>0) { i_stream_skip(is_3, amt); }
+
+        if (is_3->stream_errno != 0)
+                i_debug("error: %s", i_stream_get_error(is_3));
+
+	test_assert(is_3->stream_errno == 0);
+
+	i_stream_unref(&is_3);
+
+	hash->result(hash_ctx, hash_dgst);
+
+	test_assert(strcmp(test_sample_v2_hash, binary_to_hex(hash_dgst, sizeof(hash_dgst))) == 0);
+
+	test_end();
+
+/** this code is left here to show how the sample file is created
+	struct istream *is = i_stream_create_file("../lib-fts/udhr_fra.txt", 8192);
+	struct istream *is_2 = i_stream_create_hash(is, hash, hash_ctx);
+	int fd = open("sample-v2.bin", O_CREAT|O_TRUNC|O_WRONLY, S_IRWXU);
+	struct ostream *os = o_stream_create_fd_file(fd, 0, TRUE);
+	struct ostream *os_2 = o_stream_create_encrypt(os, "aes-256-gcm-sha256", test_v2_kp.pub, IO_STREAM_ENC_INTEGRITY_AEAD);
+	const unsigned char *ptr;
+	size_t siz;
+
+	while(i_stream_read_data(is_2, &ptr, &siz, 0)>0) {
+		o_stream_nsend(os_2, ptr, siz);
+		i_stream_skip(is_2, siz);
+	}
+
+	i_assert(o_stream_nfinish(os_2)==0);
+
+	o_stream_close(os_2);
+	i_stream_close(is_2);
+
+	hash->result(hash_ctx, hash_dgst);
+	printf("%s\n", binary_to_hex(hash_dgst, sizeof(hash_dgst)));
+*/
+}
+
+static
+void test_write_read_v1(void)
+{
+	test_begin("test_write_read_v1");
+	unsigned char payload[IO_BLOCK_SIZE];
+	const unsigned char *ptr;
+	size_t pos = 0, siz;
+	random_fill_weak(payload, IO_BLOCK_SIZE);
+
+	struct ostream *os = iostream_temp_create("/tmp", 0);
+	struct ostream *os_2 = o_stream_create_encrypt(os, "<unused>", test_v2_kp.pub, IO_STREAM_ENC_VERSION_1);
+	o_stream_nsend(os_2, payload, sizeof(payload));
+
+        if (os_2->stream_errno != 0)
+                i_debug("error: %s", o_stream_get_error(os_2));
+
+	test_assert(os_2->stream_errno == 0);
+	test_assert(o_stream_nfinish(os_2) == 0);
+	test_assert(os_2->stream_errno == 0);
+
+	o_stream_unref(&os_2);
+
+	struct istream *is = iostream_temp_finish(&os, IO_BLOCK_SIZE);
+	struct istream *is_2 = i_stream_create_decrypt(is, test_v2_kp.priv);
+	i_stream_unref(&is);
+
+	while(i_stream_read_data(is_2, &ptr, &siz, 0)>0) {
+		test_assert_idx(pos + siz <= sizeof(payload), pos);
+		if (pos + siz > sizeof(payload)) break;
+		test_assert_idx(memcmp(ptr, payload + pos, siz) == 0, pos);
+		i_stream_skip(is_2, siz);
+	}
+
+	test_assert(is_2->stream_errno == 0);
+
+	i_stream_unref(&is_2);
+
+	test_end();
+}
+
+static
+void test_write_read_v2(void)
+{
+	test_begin("test_write_read_v2");
+	unsigned char payload[IO_BLOCK_SIZE];
+	const unsigned char *ptr;
+	size_t pos = 0, siz;
+	random_fill_weak(payload, IO_BLOCK_SIZE);
+
+	struct ostream *os = iostream_temp_create("/tmp", 0);
+	struct ostream *os_2 = o_stream_create_encrypt(os, "aes-256-gcm-sha256", test_v1_kp.pub, IO_STREAM_ENC_INTEGRITY_AEAD);
+	o_stream_nsend(os_2, payload, IO_BLOCK_SIZE);
+	test_assert(o_stream_nfinish(os_2) == 0);
+        if (os_2->stream_errno != 0)
+                i_debug("error: %s", o_stream_get_error(os_2));
+
+	o_stream_unref(&os_2);
+
+	struct istream *is = iostream_temp_finish(&os, IO_BLOCK_SIZE);
+	struct istream *is_2 = i_stream_create_decrypt(is, test_v1_kp.priv);
+	i_stream_unref(&is);
+
+	while(i_stream_read_data(is_2, &ptr, &siz, 0)>0) {
+		test_assert_idx(pos + siz <= sizeof(payload), pos);
+		if (pos + siz > sizeof(payload)) break;
+		test_assert_idx(memcmp(ptr, payload + pos, siz) == 0, pos);
+		i_stream_skip(is_2, siz);
+	}
+
+	test_assert(is_2->stream_errno == 0);
+        if (is_2->stream_errno != 0)
+                i_debug("error: %s", i_stream_get_error(is_2));
+
+	i_stream_unref(&is_2);
+
+	test_end();
+}
+
+int main(void) {
+	dcrypt_initialize("openssl", NULL);
+	random_init();
+
+	dcrypt_key_load_private(&test_v1_kp.priv, DCRYPT_FORMAT_PEM, key_v1_priv, NULL, NULL, NULL);
+	dcrypt_key_load_public(&test_v1_kp.pub, DCRYPT_FORMAT_PEM, key_v1_pub, NULL);
+	dcrypt_key_load_private(&test_v2_kp.priv, DCRYPT_FORMAT_PEM, key_v2_priv, NULL, NULL, NULL);
+	dcrypt_key_load_public(&test_v2_kp.pub, DCRYPT_FORMAT_PEM, key_v2_pub, NULL);
+
+	static void (*test_functions[])(void) = {
+		test_static_v1_input,
+		test_static_v2_input,
+		test_write_read_v1,
+		test_write_read_v2,
+		NULL
+	};
+
+	return test_run(test_functions);
+}
diff --git a/src/lib-storage/index/dbox-single/sdbox-file.c b/src/lib-storage/index/dbox-single/sdbox-file.c
index 347d77f84d8..ceb60738a58 100644
--- a/src/lib-storage/index/dbox-single/sdbox-file.c
+++ b/src/lib-storage/index/dbox-single/sdbox-file.c
@@ -311,6 +311,9 @@ int sdbox_file_move(struct dbox_file *file, bool alt_path)
 	}
 
 	dest_path = !alt_path ? file->primary_path : file->alt_path;
+
+	i_assert(dest_path != NULL);
+
 	p = strrchr(dest_path, '/');
 	i_assert(p != NULL);
 	dest_dir = t_strdup_until(dest_path, p);

From 220df28477e92297c137d6fc5e3527940fcb4b85 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Fri, 6 May 2016 00:29:16 +0300
Subject: [PATCH 188/450] configure: Include lib-dcrypt in core

---
 configure.ac | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 0bcdceba293..791c960f30b 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2516,7 +2516,7 @@ dnl **
 dnl ** Shared libraries usage
 dnl **
 
-LIBDOVECOT_LA_LIBS='$(top_builddir)/src/lib-dict-extra/libdict_extra.la $(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-stats/libstats.la $(top_builddir)/src/lib-http/libhttp.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-sasl/libsasl.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la $(top_builddir)/src/lib-test/libtest.la $(top_builddir)/src/lib/liblib.la'
+LIBDOVECOT_LA_LIBS='$(top_builddir)/src/lib-dict-extra/libdict_extra.la $(top_builddir)/src/lib-master/libmaster.la $(top_builddir)/src/lib-settings/libsettings.la $(top_builddir)/src/lib-stats/libstats.la $(top_builddir)/src/lib-http/libhttp.la $(top_builddir)/src/lib-fs/libfs.la $(top_builddir)/src/lib-dict/libdict.la $(top_builddir)/src/lib-dns/libdns.la $(top_builddir)/src/lib-imap/libimap.la $(top_builddir)/src/lib-mail/libmail.la $(top_builddir)/src/lib-sasl/libsasl.la $(top_builddir)/src/lib-auth/libauth.la $(top_builddir)/src/lib-charset/libcharset.la $(top_builddir)/src/lib-ssl-iostream/libssl_iostream.la $(top_builddir)/src/lib-dcrypt/libdcrypt.la $(top_builddir)/src/lib-test/libtest.la $(top_builddir)/src/lib/liblib.la'
 if test "$want_shared_libs" = "yes"; then
   LIBDOVECOT_DEPS='$(top_builddir)/src/lib-dovecot/libdovecot.la'
   LIBDOVECOT="$LIBDOVECOT_DEPS \$(MODULE_LIBS)"

From 5a5ddc2fb3b5b0387d38b0ba27cc6ed6a8e41ef0 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 27 May 2016 22:15:07 +0300
Subject: [PATCH 189/450] lib-storage: Added MAILBOX_FLAG_DELETE_UNSAFE

---
 src/lib-storage/mail-storage.h                 | 8 +++++++-
 src/plugins/lazy-expunge/lazy-expunge-plugin.c | 2 +-
 src/plugins/quota/quota-storage.c              | 3 ++-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/src/lib-storage/mail-storage.h b/src/lib-storage/mail-storage.h
index 4c99849e50b..64804b4f334 100644
--- a/src/lib-storage/mail-storage.h
+++ b/src/lib-storage/mail-storage.h
@@ -49,7 +49,13 @@ enum mailbox_flags {
 	/* Force opening mailbox and ignoring any ACLs */
 	MAILBOX_FLAG_IGNORE_ACLS	= 0x100,
 	/* Open mailbox even if it's already marked as deleted */
-	MAILBOX_FLAG_OPEN_DELETED	= 0x200
+	MAILBOX_FLAG_OPEN_DELETED	= 0x200,
+	/* Mailbox is opened for deletion, which should be performed as
+	   efficiently as possible, even allowing the mailbox state to become
+	   inconsistent. For example this disables lazy_expunge plugin and
+	   quota updates (possibly resulting in broken quota). and This is
+	   useful for example when deleting entire user accounts. */
+	MAILBOX_FLAG_DELETE_UNSAFE	= 0x400
 };
 
 enum mailbox_feature {
diff --git a/src/plugins/lazy-expunge/lazy-expunge-plugin.c b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
index b9faf182e55..baa6ffa13be 100644
--- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c
+++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
@@ -448,7 +448,7 @@ static void lazy_expunge_mailbox_allocated(struct mailbox *box)
 	union mailbox_module_context *mbox;
 	struct mailbox_vfuncs *v = box->vlast;
 
-	if (llist == NULL)
+	if (llist == NULL || (box->flags & MAILBOX_FLAG_DELETE_UNSAFE) != 0)
 		return;
 
 	mbox = p_new(box->pool, union mailbox_module_context, 1);
diff --git a/src/plugins/quota/quota-storage.c b/src/plugins/quota/quota-storage.c
index bd65663b537..4d75a319437 100644
--- a/src/plugins/quota/quota-storage.c
+++ b/src/plugins/quota/quota-storage.c
@@ -328,7 +328,8 @@ static void quota_mailbox_sync_notify(struct mailbox *box, uint32_t uid,
 	if (qbox->module_ctx.super.sync_notify != NULL)
 		qbox->module_ctx.super.sync_notify(box, uid, sync_type);
 
-	if (sync_type != MAILBOX_SYNC_TYPE_EXPUNGE || qbox->recalculate) {
+	if (sync_type != MAILBOX_SYNC_TYPE_EXPUNGE || qbox->recalculate ||
+	    (box->flags & MAILBOX_FLAG_DELETE_UNSAFE) != 0) {
 		if (uid == 0) {
 			/* free the transaction before view syncing begins,
 			   otherwise it'll crash. */

From 69cf950e72517e6fb0e2b755c2ee0386fc8f06f9 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 27 May 2016 22:18:17 +0300
Subject: [PATCH 190/450] doveadm mailbox delete: Added --unsafe option.

---
 src/doveadm/doveadm-mail-mailbox.c | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/doveadm/doveadm-mail-mailbox.c b/src/doveadm/doveadm-mail-mailbox.c
index 19ef531480d..d99e45effd5 100644
--- a/src/doveadm/doveadm-mail-mailbox.c
+++ b/src/doveadm/doveadm-mail-mailbox.c
@@ -35,6 +35,7 @@ struct delete_cmd_context {
 	ARRAY_TYPE(const_string) mailboxes;
 	bool recursive;
 	bool require_empty;
+	bool unsafe;
 };
 
 struct rename_cmd_context {
@@ -335,8 +336,11 @@ cmd_mailbox_delete_run(struct doveadm_mail_cmd_context *_ctx,
 	const char *const *namep;
 	ARRAY_TYPE(const_string) recursive_mailboxes;
 	const ARRAY_TYPE(const_string) *mailboxes = &ctx->mailboxes;
+	enum mailbox_flags mailbox_flags = 0;
 	int ret = 0, ret2;
 
+	if (ctx->unsafe)
+		mailbox_flags |= MAILBOX_FLAG_DELETE_UNSAFE;
 	if (ctx->recursive) {
 		t_array_init(&recursive_mailboxes, 32);
 		array_foreach(&ctx->mailboxes, namep) {
@@ -356,7 +360,7 @@ cmd_mailbox_delete_run(struct doveadm_mail_cmd_context *_ctx,
 		const char *name = *namep;
 
 		ns = mail_namespace_find(user->namespaces, name);
-		box = mailbox_alloc(ns->list, name, 0);
+		box = mailbox_alloc(ns->list, name, mailbox_flags);
 		storage = mailbox_get_storage(box);
 		ret2 = ctx->require_empty ? mailbox_delete_empty(box) :
 			mailbox_delete(box);
@@ -412,6 +416,9 @@ cmd_mailbox_delete_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
 	case 'e':
 		ctx->require_empty = TRUE;
 		break;
+	case 'Z':
+		ctx->unsafe = TRUE;
+		break;
 	default:
 		return FALSE;
 	}
@@ -426,7 +433,7 @@ static struct doveadm_mail_cmd_context *cmd_mailbox_delete_alloc(void)
 	ctx->ctx.ctx.v.init = cmd_mailbox_delete_init;
 	ctx->ctx.ctx.v.run = cmd_mailbox_delete_run;
 	ctx->ctx.ctx.v.parse_arg = cmd_mailbox_delete_parse_arg;
-	ctx->ctx.ctx.getopt_args = "ers";
+	ctx->ctx.ctx.getopt_args = "ersZ";
 	p_array_init(&ctx->mailboxes, ctx->ctx.ctx.pool, 16);
 	return &ctx->ctx.ctx;
 }
@@ -687,12 +694,13 @@ DOVEADM_CMD_PARAMS_END
 struct doveadm_cmd_ver2 doveadm_cmd_mailbox_delete_ver2 = {
 	.name = "mailbox delete",
 	.mail_cmd = cmd_mailbox_delete_alloc,
-	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-e] [-r] [-s] <mailbox> [...]",
+	.usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-e] [-r] [-s] [-Z] <mailbox> [...]",
 DOVEADM_CMD_PARAMS_START
 DOVEADM_CMD_MAIL_COMMON
 DOVEADM_CMD_PARAM('e', "require-empty", CMD_PARAM_BOOL, 0)
 DOVEADM_CMD_PARAM('s', "subscriptions", CMD_PARAM_BOOL, 0)
 DOVEADM_CMD_PARAM('r', "recursive", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('Z', "unsafe", CMD_PARAM_BOOL, 0)
 DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
 DOVEADM_CMD_PARAMS_END
 };

From a991c2181600f6d9cf40c94a6e90acbea2ae1d28 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 30 May 2016 15:36:17 +0300
Subject: [PATCH 191/450] lib-dcrypt: Compiler warning fixes

---
 src/lib-dcrypt/dcrypt-openssl.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index d60c5a4da23..60cb0a9de36 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -1647,8 +1647,8 @@ bool dcrypt_openssl_key_string_get_info(const char *key_data, enum dcrypt_key_fo
 	enum dcrypt_key_version version = DCRYPT_KEY_VERSION_NA;
 	enum dcrypt_key_encryption_type encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_NONE;
 	enum dcrypt_key_kind kind = DCRYPT_KEY_KIND_PUBLIC;
-	const char *encryption_key_hash = NULL;
-	const char *key_hash = NULL;
+	char *encryption_key_hash = NULL;
+	char *key_hash = NULL;
 
 	if (key_data == NULL) {
 		if (error_r != NULL)

From 3dfcebbc15ac8fc0c0f95c1de4a98bf67db6fd69 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 30 May 2016 19:38:22 +0300
Subject: [PATCH 192/450] lib-dcrypt: sample-v2.bin shouldn't be executable

---
 src/lib-dcrypt/sample-v2.bin | Bin
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100755 => 100644 src/lib-dcrypt/sample-v2.bin

diff --git a/src/lib-dcrypt/sample-v2.bin b/src/lib-dcrypt/sample-v2.bin
old mode 100755
new mode 100644

From 619830d4c0c45925c140889d9a1fc0fa86d13afc Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 30 May 2016 21:14:53 +0300
Subject: [PATCH 193/450] lib-dcrypt: Always allow error_r to be NULL.

Especially dcrypt_openssl_private_to_public_key() was called with
error_r=NULL by the dcrypt.c itself.
---
 src/lib-dcrypt/dcrypt-openssl.c | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index 60cb0a9de36..b477a1baee4 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -385,7 +385,8 @@ bool dcrypt_openssl_ctx_hmac_create(const char *algorithm, struct dcrypt_context
 	const EVP_MD *md;
 	md = EVP_get_digestbyname(algorithm);
 	if(md == NULL) {
-		*error_r = t_strdup_printf("Invalid digest %s", algorithm);
+		if (error_r != NULL)
+			*error_r = t_strdup_printf("Invalid digest %s", algorithm);
 		return FALSE;
 	}
 	/* allocate context */
@@ -673,7 +674,8 @@ bool dcrypt_openssl_pbkdf2(const unsigned char *password, size_t password_len, c
 		/* determine MD */
 		const EVP_MD* md = EVP_get_digestbyname(hash);
 		if (md == NULL) {
-			*error_r = t_strdup_printf("Invalid digest %s", hash);
+			if (error_r != NULL)
+				*error_r = t_strdup_printf("Invalid digest %s", hash);
 			return FALSE;
 		}
 
@@ -1550,7 +1552,8 @@ bool dcrypt_openssl_store_private_key(struct dcrypt_private_key *key, enum dcryp
 	if (cipher != NULL) {
 		algo = EVP_get_cipherbyname(cipher);
 		if (algo == NULL) {
-			*error_r = t_strdup_printf("Invalid cipher %s", cipher);
+			if (error_r != NULL)
+				*error_r = t_strdup_printf("Invalid cipher %s", cipher);
 			return FALSE;
 		}
 	}
@@ -1630,7 +1633,8 @@ bool dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, s
 		EVP_PKEY_set1_EC_KEY(pk, eck);
 		EC_KEY_free(eck);
 	} else {
-		*error_r = "Invalid private key";
+		if (error_r != NULL)
+			*error_r = "Invalid private key";
 		return FALSE;
 	}
 

From 2c6877f5f8979747384f50fda82c56cb1f0b249e Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Mon, 30 May 2016 22:05:22 +0300
Subject: [PATCH 194/450] lib-dcrypt: Remove duplication of samples

---
 src/lib-dcrypt/Makefile.am | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/lib-dcrypt/Makefile.am b/src/lib-dcrypt/Makefile.am
index 73c5c1d9a98..98666d8c99f 100644
--- a/src/lib-dcrypt/Makefile.am
+++ b/src/lib-dcrypt/Makefile.am
@@ -45,11 +45,6 @@ check-test: all-am
 	  if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
 	done
 
-
-noinst_DATA = \
-	sample-v1.bin \
-	sample-v2.bin
-
 LIBDOVECOT_TEST = \
 	../lib-test/libtest.la \
 	../lib/liblib.la \

From 7251106030d7ce8cb91aff117e349d10da6de1c1 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Mon, 30 May 2016 22:30:31 +0300
Subject: [PATCH 195/450] lib-dcrypt: Fix various OpenSSL API usage issues

---
 src/lib-dcrypt/dcrypt-openssl.c | 47 ++++++++++++++++++---------------
 1 file changed, 26 insertions(+), 21 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index b477a1baee4..aab9ce2284e 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -66,6 +66,11 @@
   2<tab>key algo oid<tab>1<tab>symmetric algo name<tab>salt<tab>hash algo<tab>rounds<tab>E(RSA = i2d_PrivateKey, EC=Private Point)<tab>key id
 **/
 
+#if SSLEAY_VERSION_NUMBER < 0x1010000fL
+#define EVP_PKEY_get0_EC_KEY(x) x->pkey.ec
+#define EVP_PKEY_get0_RSA(x) x->pkey.rsa
+#endif
+
 struct dcrypt_context_symmetric {
 	pool_t pool;
 	const EVP_CIPHER *cipher;
@@ -522,8 +527,8 @@ bool dcrypt_openssl_generate_ec_key(int nid, EVP_PKEY **key, const char **error_
 	EVP_PKEY_free(params);
 	EVP_PKEY_CTX_free(pctx);
 	EVP_PKEY_CTX_free(ctx);
-	EC_KEY_set_asn1_flag((*key)->pkey.ec, OPENSSL_EC_NAMED_CURVE);
-	EC_KEY_set_conv_form((*key)->pkey.ec, POINT_CONVERSION_COMPRESSED);
+	EC_KEY_set_asn1_flag(EVP_PKEY_get0_EC_KEY((*key)), OPENSSL_EC_NAMED_CURVE);
+	EC_KEY_set_conv_form(EVP_PKEY_get0_EC_KEY((*key)), POINT_CONVERSION_COMPRESSED);
 	return TRUE;
 }
 
@@ -551,7 +556,7 @@ bool dcrypt_openssl_ecdh_derive_secret_local(struct dcrypt_private_key *local_ke
 {
 	EVP_PKEY *local = (EVP_PKEY*)local_key;
 	BN_CTX *bn_ctx = BN_CTX_new();
-	const EC_GROUP *grp = EC_KEY_get0_group(local->pkey.ec);
+	const EC_GROUP *grp = EC_KEY_get0_group(EVP_PKEY_get0_EC_KEY(local));
 	EC_POINT *pub = EC_POINT_new(grp);
 	/* convert ephemeral key data EC point */
 	if (EC_POINT_oct2point(grp, pub, R->data, R->used, bn_ctx) != 1)
@@ -621,7 +626,7 @@ bool dcrypt_openssl_ecdh_derive_secret_peer(struct dcrypt_public_key *peer_key,
 	}
 
 	/* generate another key from same group */
-	int nid = EC_GROUP_get_curve_name(EC_KEY_get0_group(peer->pkey.ec));
+	int nid = EC_GROUP_get_curve_name(EC_KEY_get0_group(EVP_PKEY_get0_EC_KEY(peer)));
 	if (!dcrypt_openssl_generate_ec_key(nid, &local, error_r)) return FALSE;
 
 	/* initialize */
@@ -650,8 +655,8 @@ bool dcrypt_openssl_ecdh_derive_secret_peer(struct dcrypt_public_key *peer_key,
 
 	/* get ephemeral key (=R) */
 	BN_CTX *bn_ctx = BN_CTX_new();
-	const EC_POINT *pub = EC_KEY_get0_public_key(local->pkey.ec);
-	const EC_GROUP *grp = EC_KEY_get0_group(local->pkey.ec);
+	const EC_POINT *pub = EC_KEY_get0_public_key(EVP_PKEY_get0_EC_KEY(local));
+	const EC_GROUP *grp = EC_KEY_get0_group(EVP_PKEY_get0_EC_KEY(local));
 	len = EC_POINT_point2oct(grp, pub, POINT_CONVERSION_COMPRESSED, NULL, 0, bn_ctx);
 	unsigned char R_buf[len];
 	EC_POINT_point2oct(grp, pub, POINT_CONVERSION_COMPRESSED, R_buf, len, bn_ctx);
@@ -1350,7 +1355,7 @@ bool dcrypt_openssl_store_private_key_dovecot(struct dcrypt_private_key *key, co
 	ASN1_OBJECT *obj;
 	if (EVP_PKEY_base_id(pkey) == EVP_PKEY_EC) {
 		/* because otherwise we get wrong nid */
-		obj = OBJ_nid2obj(EC_GROUP_get_curve_name(EC_KEY_get0_group(pkey->pkey.ec)));
+		obj = OBJ_nid2obj(EC_GROUP_get_curve_name(EC_KEY_get0_group(EVP_PKEY_get0_EC_KEY(pkey))));
 
 	} else {
 		obj = OBJ_nid2obj(EVP_PKEY_id(pkey));
@@ -1371,14 +1376,14 @@ bool dcrypt_openssl_store_private_key_dovecot(struct dcrypt_private_key *key, co
 	/* convert key to private key value */
 	if (EVP_PKEY_base_id(pkey) == EVP_PKEY_RSA) {
 		unsigned char *ptr;
-		RSA *rsa = pkey->pkey.rsa;
+		RSA *rsa = EVP_PKEY_get0_RSA(pkey);
 		int ln = i2d_RSAPrivateKey(rsa, &ptr);
 		if (ln < 1)
 			return dcrypt_openssl_error(error_r);
 		buffer_append(buf, ptr, ln);
 	} else if (EVP_PKEY_base_id(pkey) == EVP_PKEY_EC) {
 		unsigned char *ptr;
-		EC_KEY *eckey = pkey->pkey.ec;
+		EC_KEY *eckey = EVP_PKEY_get0_EC_KEY(pkey);
 		const BIGNUM *pk = EC_KEY_get0_private_key(eckey);
 		/* serialize to MPI which is portable */
 		int len = BN_bn2mpi(pk, NULL);
@@ -1480,7 +1485,7 @@ bool dcrypt_openssl_load_private_key(struct dcrypt_private_key **key_r, enum dcr
 	}
 
 	if (EVP_PKEY_base_id(key) == EVP_PKEY_EC) {
-		EC_KEY_set_conv_form(key->pkey.ec, POINT_CONVERSION_COMPRESSED);
+		EC_KEY_set_conv_form(EVP_PKEY_get0_EC_KEY(key), POINT_CONVERSION_COMPRESSED);
 	}
 
 	*key_r = (struct dcrypt_private_key *)key;
@@ -1499,7 +1504,7 @@ bool dcrypt_openssl_load_public_key(struct dcrypt_public_key **key_r, enum dcryp
 	BIO *key_in = BIO_new_mem_buf((void*)data, strlen(data));
 
 	key = PEM_read_bio_PUBKEY(key_in, &key, NULL, NULL);
-	BIO_reset(key_in);
+	(void)BIO_reset(key_in);
 	if (key == NULL) { /* ec keys are bother */
 		/* read the header */
 		char buf[27]; /* begin public key */
@@ -1560,7 +1565,7 @@ bool dcrypt_openssl_store_private_key(struct dcrypt_private_key *key, enum dcryp
 
 	ec = PEM_write_bio_PrivateKey(key_out, pkey, algo, NULL, 0, NULL, (void*)password);
 
-	BIO_flush(key_out);
+	(void)BIO_flush(key_out);
 
 	if (ec != 1) {
 		BIO_vfree(key_out);
@@ -1590,13 +1595,13 @@ bool dcrypt_openssl_store_public_key(struct dcrypt_public_key *key, enum dcrypt_
 		ec = PEM_write_bio_PUBKEY(key_out, pkey);
 	else {
 		BIO *b64 = BIO_new(BIO_f_base64());
-		BIO_puts(key_out, "-----BEGIN PUBLIC KEY-----\n");
-		BIO_push(b64, key_out);
-		ec = i2d_EC_PUBKEY_bio(b64, pkey->pkey.ec);
-		BIO_flush(b64);
-		BIO_pop(b64);
+		(void)BIO_puts(key_out, "-----BEGIN PUBLIC KEY-----\n");
+		(void)BIO_push(b64, key_out);
+		ec = i2d_EC_PUBKEY_bio(b64, EVP_PKEY_get0_EC_KEY(pkey));
+		(void)BIO_flush(b64);
+		(void)BIO_pop(b64);
 		BIO_vfree(b64);
-		BIO_puts(key_out, "-----END PUBLIC KEY-----");
+		(void)BIO_puts(key_out, "-----END PUBLIC KEY-----");
 	}
 
 	if (ec != 1) {
@@ -1626,7 +1631,7 @@ bool dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, s
 
 	if (EVP_PKEY_base_id(pkey) == EVP_PKEY_RSA)
 	{
-		EVP_PKEY_set1_RSA(pk, RSAPublicKey_dup(pkey->pkey.rsa));
+		EVP_PKEY_set1_RSA(pk, RSAPublicKey_dup(EVP_PKEY_get0_RSA(pkey)));
 	} else if (EVP_PKEY_base_id(pkey) == EVP_PKEY_EC) {
 		EC_KEY* eck = EVP_PKEY_get1_EC_KEY(pkey);
 		EC_KEY_set_asn1_flag(eck, OPENSSL_EC_NAMED_CURVE);
@@ -1891,7 +1896,7 @@ bool dcrypt_openssl_public_key_id_old(struct dcrypt_public_key *key, buffer_t *r
 		return FALSE;
 	}
 
-	char *pub_pt_hex = ec_key_get_pub_point_hex(pub->pkey.ec);
+	char *pub_pt_hex = ec_key_get_pub_point_hex(EVP_PKEY_get0_EC_KEY(pub));
 	/* digest this */
 	SHA256((const unsigned char*)pub_pt_hex, strlen(pub_pt_hex), buf);
 	buffer_append(result, buf, SHA256_DIGEST_LENGTH);
@@ -1919,7 +1924,7 @@ bool dcrypt_openssl_public_key_id(struct dcrypt_public_key *key, const char *alg
 		return FALSE;
 	}
 	if (EVP_PKEY_base_id(pub) == EVP_PKEY_EC) {
-		EC_KEY_set_conv_form(pub->pkey.ec, POINT_CONVERSION_COMPRESSED);
+		EC_KEY_set_conv_form(EVP_PKEY_get0_EC_KEY(pub), POINT_CONVERSION_COMPRESSED);
 	}
 	BIO *b = BIO_new(BIO_s_mem());
 	if (i2d_PUBKEY_bio(b, pub) < 1) {

From fe60fa9da879048185e1b68729215057a73fefa2 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Mon, 30 May 2016 22:33:29 +0300
Subject: [PATCH 196/450] lib-dcrypt: Fix strict type-punning warning

---
 src/lib-dcrypt/dcrypt-openssl.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index aab9ce2284e..d51233fa6cc 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -697,8 +697,10 @@ bool dcrypt_openssl_pbkdf2(const unsigned char *password, size_t password_len, c
 static
 bool dcrypt_openssl_generate_keypair(struct dcrypt_keypair *pair_r, enum dcrypt_key_type kind, unsigned int bits, const char *curve, const char **error_r)
 {
+	EVP_PKEY *pkey = NULL;
 	if (kind == DCRYPT_KEY_RSA) {
-		if (dcrypt_openssl_generate_rsa_key(bits, (EVP_PKEY**)&(pair_r->priv), error_r) == 0) {
+		if (dcrypt_openssl_generate_rsa_key(bits, &pkey, error_r) == 0) {
+			pair_r->priv = (struct dcrypt_private_key*)pkey;
 			return dcrypt_openssl_private_to_public_key(pair_r->priv, &(pair_r->pub), error_r);
 		} else return dcrypt_openssl_error(error_r);
 	} else if (kind == DCRYPT_KEY_EC) {
@@ -708,7 +710,8 @@ bool dcrypt_openssl_generate_keypair(struct dcrypt_keypair *pair_r, enum dcrypt_
 				*error_r = t_strdup_printf("Unknown EC curve %s", curve);
 			return FALSE;
 		}
-		if (dcrypt_openssl_generate_ec_key(nid, (EVP_PKEY**)&(pair_r->priv), error_r) == 0) {
+		if (dcrypt_openssl_generate_ec_key(nid, &pkey, error_r) == 0) {
+			pair_r->priv = (struct dcrypt_private_key*)pkey;
 			return dcrypt_openssl_private_to_public_key(pair_r->priv, &(pair_r->pub), error_r);
 		} else return dcrypt_openssl_error(error_r);
 	}

From d26c53ce49c343c841c1c4f69784e22300edd8d8 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Mon, 30 May 2016 22:39:39 +0300
Subject: [PATCH 197/450] lib-dcrypt: Use base64 encoded test samples

---
 src/lib-dcrypt/Makefile.am   |   4 +-
 src/lib-dcrypt/sample-v1.asc |  48 ++++++
 src/lib-dcrypt/sample-v1.bin | Bin 2281 -> 0 bytes
 src/lib-dcrypt/sample-v2.asc | 287 +++++++++++++++++++++++++++++++++++
 src/lib-dcrypt/sample-v2.bin | Bin 13757 -> 0 bytes
 src/lib-dcrypt/test-stream.c |  37 +++--
 6 files changed, 358 insertions(+), 18 deletions(-)
 create mode 100644 src/lib-dcrypt/sample-v1.asc
 delete mode 100644 src/lib-dcrypt/sample-v1.bin
 create mode 100644 src/lib-dcrypt/sample-v2.asc
 delete mode 100644 src/lib-dcrypt/sample-v2.bin

diff --git a/src/lib-dcrypt/Makefile.am b/src/lib-dcrypt/Makefile.am
index 98666d8c99f..5f24f3ea518 100644
--- a/src/lib-dcrypt/Makefile.am
+++ b/src/lib-dcrypt/Makefile.am
@@ -33,8 +33,8 @@ pkginc_libdir=$(pkgincludedir)
 pkginc_lib_HEADERS = $(headers)
 
 EXTRA_DIST = \
-	sample-v1.bin \
-	sample-v2.bin
+	sample-v1.asc \
+	sample-v2.asc
 
 check_PROGRAMS = test-crypto test-stream
 
diff --git a/src/lib-dcrypt/sample-v1.asc b/src/lib-dcrypt/sample-v1.asc
new file mode 100644
index 00000000000..c69fb509546
--- /dev/null
+++ b/src/lib-dcrypt/sample-v1.asc
@@ -0,0 +1,48 @@
+Q1JZUFRFRAMHAQCtAEMCAMyKuGO/j3TPoXzRJ39THa1oGDChmueqRVFlR3qnIWcd
+Mt15zp+juTJzwfxKDNsgdIfFleIbuuo1AX1TgimaVfb8ACB8mhA56i5P7XPoHdP/
+w/oi6kooNSk5rd57+OqFiwD6TwAgWi/IHZ3tFmaohetUkFowgcrYwMh9HR9iOXg6
+QdIDnqMAIIrC9OcLyuMzUp18LpVKZLg6QaJEsjrepBatgkqRDgBKAAD7tnI+Rjjg
+rNZ5UHYvjA1xsrEhfbyx8X6Vb+em++p+aE+I92pBrqV/XIeR1er1oNX3nxZwEnL4
+UwhavZOMw7Qna0o4bop4PfK65HqnFTmgaiNDBMdE/CFaxSRlI0PLc0jhqxoEYU/d
+0hrtPcpQMq0sCxBbHqcnDF1xAK2hAZU12BH+JoV4bI1k1MMn1xAcXxiVdtSO5NE8
+E5fyaMfvTq2zIZtqQY09arrd0DaQ8o/L2dIV6jQVNZLxbRFWVayoWloT/YVSlHhi
+w5YHJetffXO02mllj3Mxr5aIfCmpZbrcWMfrF88ksI6HyQOrTHKS+Y95+VFLbxy4
++BWFGKV4zUGUwrfEDOpxIAZbBHsABjV82NS2TEZltu/ki3EhlwC8Hy+oyqn+LZN5
+LCmbt/maMI0EJU6cNCUCQM8Rq2Xv7xP/DrC3A0y7gj3pT44dY0dMj76gBApKO4Qw
+rLcBgY9qycXHtUwvtg/QZJrb2n2AB7h0+B3LgVm8P12l1KAFS2ykugBWUVJSuUjK
+5fjTk4EDIaCm0rhs2hNty9OtBkBuQBolkzxHtqp9/+QhIWJEtNtQEdnwN8DtdlNH
+/p69sZvkDhmyqSuByCodwVhkPZf5d/oGVUvE73btlAJcl8NZMXDqgKHfT4U5OF6Y
+eSXUIz9oiM2Wy1CVqrA2JdFGQ4Hcbf76IP462+gGefOd62atFfvjGGIb+Okyrmab
+jNxn/7wtUw42MXIzoAk+GQsOgo77rH075mILWqp0OuAyRTKmsZTP3FaDb4SxQk3K
+pw3N7HiNIiDW1wd5BLJE73qxKr+JC8GLs/s+JpfVb+lxzXAKXpEZTGFd/zphF1Kf
+J+aBCF+UB5Lq+QYoHfKSJzRb24PScAVs1VrV8nJlQvxMZW6Rd4ofNcsMjY+b1pBx
+bv74+oACEXC/F1jpp3tMTTLOLN8/hNq0J1mE3uOYRAaNK1jaQQAoa4rNxeY/2vBw
+mod8/Erc9M0M4B3VHVfRJ4F8MAx3b8if3oiicLu0OJ16snGHM7BGp2pOEPRVAg1Q
++70zDctuSV6HwPYSAXaIw2LPrbsQmq1Lq/JivHYRwjUDBFAaXkPv2WFEfUsXEtmw
+UzUPLgWxZM8MsPDEI510SzRd3OlBUjuS96+/9n/Qv1D5DBnrDH9gUwUiPr9V+rMM
+kmNaAQpTKpCNpevCH/F18rQnUi/PJKY37q8pF/lO0OW7mzS+CxCvFr+aWxZ2kV84
+GwUrUuSdHa1Z/0BY/UTZn1BAi6bOudiWHH5loJBldreSJ2K2/iwMpLUUEuQ+TJJ8
+BvsWeOLMqEpDIxuuYERCUqHO8EmtvsSPKcIeS7ZZpvkRlRIWCuKgnaHSymT87Eq1
+SJskTkAWd1iGvQI78tM8KwT2KDmf7Qs3RUCiSynT/H1OmVQqwVn/5c9wFjV+0PRA
+KFZZDEMvwy5tYpJ1Y0nYuUUMOlA8l11rpKqAIcRj0V256uXoj5cBnTsGZAOFDBqx
+pBLmGFgz1havj8RsgqtJfCkkh+Y8l9xszAC98/FGYOtmKpP4sXXM3LASkhi2FU4C
+OH/4tUsoh3tMCendx36s1UmliE94BiNuJMUAqSh9cLCnW/Uiw0bzqV8GOJLDk/A3
+XMWrERuA2Jn7qQ9e9qmYarc8r6JjWUuxECZ04d12wNCcDF7hWxFYLbwTk/ZUIM5D
+1ZsdOPUQDf7gjEW4gQoOK7pQWifJa56ZSSFurLoL5ae+3dPwUu8HNQLvwmx4paSe
+01shQd36MXcRZ7BRVp0GqNrviuAtXacSxx1GIO9rh7RtyGGs1grsQ07P6Evpk0k/
+1WY48cE+xWU5SH4JwxMZ3vbDguMY/cnp2VhuzZguJ4iIFKg5RMVShrSkZQcWwH7e
+7JVu7hOe3bWp1KeVG41IsOFpo0Jfpegtgf4r1hYih02Q54UNIFf5G3IRsbC1pjtj
+ALYteCLe9oa+7lIAVDWgmq/NqWLsi4dtlz97TG8XApIFZ6Prr7KG8N+RFTmouXYT
+QH6FuF5XvJ0TqIgIkdSzbuaNmN47E6PQoDAuRJ4X9ahYpF1xC4ecnAaI+rlaparF
+Z1OOtwYVQ1Hthc7wp+205aK6ujZyU6a9MrGXGZklRQdigCd+EOs0kWy8t+bcmk6K
+9aXEJyGMBtcgQDGWZZJer5w8aUKj6SDp9y7X1kyAuhh8DFvNKMgR3vs4wnvsjr1W
+cYxH8RvAVXj76Xjvvgeg3jcJbcr0mPwB0SrcpQ+88mF64x+nrKsXmz9E8wBgOMY6
+4qXXb27YUehAGN81jTZRN8lShl20fnPZKd5U1sqIhevXNUrTVjxVNff0fHRnOOb9
+sVD3vdQmfbXvB3nyQLKQ+Wcw/WPqa0But5KEXFjnaXPORcEWEpYvWOgGPegmrTgh
+P4ZVscRdxozvd0sKvpLNMd/EiNLBaYft+4Yo38WZYkZzNJT7LhW41pbQyK1cmZiU
+COisSP2rrus6iKCNwaGPMZJNCCdQN862DSTaa0IZSEeSRpfBfA2UevSsfX0uTgyb
+B4Az1u4QJv9fQp8oAnnpH7YHhW43V/YTVuwLLF7pqSl2J2gNcLFfGuJVftRp6xJg
+mkfXOxoJ0y5CQU7pwQZ/F6WxuWBX1m7MNE/OfI9l9ODj0uAgKn1e+DzH1VXZi8ls
+lenQuepkt7zqXG5dRRhW+FSlJy1oo9oPcf2bZhAYMty3mh9F4Ils3SSCE6T6cU0A
+yYCNrq8iKyq5V1jdC9haN5NmF9yNiIxYWdbigcBTzR5+AZuNe7aJSXu0qvQJR9s0
+d/l7J6LsH25I/zsIV/0OHcrvMUEf2WzU3Q==
diff --git a/src/lib-dcrypt/sample-v1.bin b/src/lib-dcrypt/sample-v1.bin
deleted file mode 100644
index e43e95b518a27a95de1758f398e6fc4282c1c046..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 2281
zcmV<F2p0E4Qdv+`MMMJ!0RXK4LjnNIinwFHk95zWe9<R=Qyr~n7%-ul=c+|fWk-6a
zA!i*j-FeQRqq#D3!Td@L+aPp@#g*b4y6QCneN%!dnpO7v03dvt5IO2DPwjK)9n=5A
z`XcH|C^acLt=@b1>V=B{`cD8LS}({Qo$VH8sD<lPkXkT-%GkijeH|ZSIe0oj(gU8O
z03eFO^ydr8<1<p7d@hwrWVkv(qC~Pf-lP_-f=ZDN07?J=`?hjEMmXTC)_G8NFN_Uw
zvaun3ys`0qm2c;!`|5sZPl)$wL9V5LT!)d>>h+-2_n#JU5_0%c2wJ_9jKj1iYf3n7
zig-Qpy5xGN6*-`4BSQqoMEoII#Uy1TL(6kW;j0=1VNc!C8tpyGP%^D73lLi#rzZ?u
zaR9BM0hKk_5&kBHcx;Vi)Wav&5FB3^m3GvQ<k376m-1-G?@q0=A)9JJjXi3*-Ox6W
z@{h~e(iQ476*ZFaZ4p*gtf*RA6a9r!lz3vpmIo#4Uww15+G%Bvb1|=$h<qujWxCv0
z$Lkl*B(RQ$$pfoQa+3LvdHGRGZydPz6@?h3c+EkS!nec>>Tw_jTLgOm1~q)x)V54U
zWw!6+i*X^B0K6YBsLHASEt7dHDVw+XnlOz7B~F|)B?3Ut5vyhI?-TzHu(tzDyMjIG
zPmUd9M@)~tpacp^JA^Q-w*i5VYRSdNwM;Lz571<q+uD7A2e@?j9m|1Pygyx~)Sv}R
zY^1sXR#8$?xk$?8_|ubt10kTM(ztBe6K%`Wtp-4DKpG{JJV&;wegEVkA!0<d+fWhN
z@HfEic2h_Gp1rY~<PI6KsVjlVDjmUCWIdPpclriZOT_PX?UVvsm%~{xaO!}e-%o`(
zI9`}}CDbE7Xo$_0%TSf7ur?*pMni$zZT|Wo{yN*}26^+H>t?ML`{Nj58~Eumu4bEz
z+-Lv1EmICQF>*7Y2|gJM4uX#RtbIG?VhdWTbUNTNMKY$bl+WB&gKvbfLQTr24bAL$
zjUph{*9Un7vPAECu`0ib3&D%C`#vU@)o<x>&2S1{kr_;3UH>{^7gC=m=79)bln0XP
z`35K*@{%VsTib)ua0P7DTGjG$WkUQ+Wp0soiXS!042_SQ){t>-{`mTU0ugY(7g*`1
zdrVC-&Me<Qgxa(xS%lu>m_!DRD_Gh=04QsU&Bf+F+VF6ihkX1>-1N;1;2qT+SJ5Yd
zd@u}mZ^)nCh@x=2v^br5vT=tqutuk9P7w4}0u506y)zBVZb@E;!1fXWc8J4b&#k)<
znypK#@?yMp5yCYC1W+1YL+{yPM14ya64|g*H4iQYv1HE-u<*nqopehyUEJwGQah6O
zufO(x(7#am3>oVTe_&GuB0j%W`m+p@V_E?UQ!0>+rR%~U@pbaFCsHrZB&Ij+uPGP#
zPSEAMn>4-)5U&=$np+ljkzY6)1uIhIogJ-N|3Fy%MA@HEK#Qi%x!9H*er2GLWp=lc
zCt|k#EDWTz6cXe<Op<&C`xbcO%&1C3BO9(@L_$)b&hSaCzQm6y!X8VuS*H0Bl@b;T
z;-H<O(#mA~>`Jvrn<P#^7I#>Ny#hP((>yB#_9!`@?F%<WK%z@2)BJr-nN%vlS^wqF
za27Rw(DXnkR#^-~FT*ZvVv==ZN!YnX3_4IemtAY5s(>NHW6@o?>gDK<mjRtS24n+;
z3>vYd66P3KGu9TbkHl<(t4Vw*B!}iam)va30KN0^Mquk^DwFuJb<Etb5|S9U6;1*;
zfB3aaD2IDY3F+O(eyr6=rHD^>1|x1H#Q>=&eQ>a+TlFHtM)Rp(1~`(#lkhiO#j6n;
zfY_P)sSjTEshDcFJg=f-Sxd1HCUoK5cEHe_3|`?|5m+s}6O;B-AkIV8n;kgy5Dos|
zj77MC3Jxo}P+BL+Yo3`&A#SX?3+1Q2-P7<=?*}yk@4{?&rKFzITOmQ+`Z0GAXRuLL
zod&4d?~33pU8fSq9Y!GUYlpOL$YHG33hYBp&*)3(lSx0-W;pS|KE-7@NPY>!6B*w2
z!-C@&{mJRsSZ>XjE+>eH6sS2w#Zrc}q-6&dz<%EBm2U16p53*n)TfmjjYzQJX`@14
zrRXhz{wvlNB8N?o=Y<U*SNR)q5wWnfraNN*wk>!f-u8yR?ot3$HK3ZW&8cGSi-&EO
zKYL7X7Xp$6XQS({vWD>Akrg?pxposkeucPRSG=7QsE7!W)U$5pjhNm$6Qj_eFfK%%
z7xk!Eq+M|fhn$=Si2Au&rK-heQ;xR=6+=<&h0gG&?X=~hy1F)UQ>MK#v6mT{B}E5f
zfG2(s>ok#Uytn4unof%KrNk#8j0V>rKrxnOl3uTzJZVCs=^*L%F4xvffVvob3|q}8
z$PwQAIKq4Ej=ffKj7RYsz*Ttr>3HwH2cX_J32n;snEV0JD%_<Hyz*gs;~%H2s~4L;
zMDqY(IL12SrPpt6*iq;}7~eIGHc>apQiff$eskF=-c;7gh=uFdHA>S~JXJOK^n7$@
zIOhGaQ1`vmCVjQ<2YK>9vXJ>_F#TidYd~(dl7w7X=V^1!MZp#lmM>W720iE|tvDe+
zhE=h|UB-;>cS{PslFc#S#E8<tX@~9mhA7{~nPNtBG?e=;6}Z-x(8#S^nV6Ia=&VTn
ztFG%hh@g$Zp^q_=O$aAYH_o;VB-(328AwNxMwh{S4U~HHtbKhhP7Ip|fHT(a5GMa$
zLZ2uCdFdav2Ze4oSN0QD><cVj>8UApCuj|Dv0obERescI>k?p^N7p+V3DYh@K~Cwx
z27ec&vAJMZ)^5x+PtJUgW%S_V(%>K}eO~xH$JJHYi^*)2>Cn0AWVgKPTy9-O7*_aH
zr6(<DquLK~{hMYG7&6?qnjb~riEP~@f)k|raZLcpfQ_!NA}cDnS6JN(*jhJ}W*6Lz
zh>TcS*5ZM{Q_UWJ0h^6`wuwo5w5s$8N82=a`Fkg#>>qAO|2qg*{SF<<?=e9i*=*F^
D+SXiI

diff --git a/src/lib-dcrypt/sample-v2.asc b/src/lib-dcrypt/sample-v2.asc
new file mode 100644
index 00000000000..0f8adaaa4b2
--- /dev/null
+++ b/src/lib-dcrypt/sample-v2.asc
@@ -0,0 +1,287 @@
+Q1JZUFRFRAMHAgAAAAIAAADPBglghkgBZQMEAS4GCWCGSAFlAwQCAQAACAAAAACf
+AQKrE9JRl23tq1RrZzVOdniCF0DdU0t0nChX9mv2K7rd/QAAACEDcgFCuwpazn7Y
+8fjHjMbv0F5XaD+Ao/EO8Kr94qRnDGcAAAAwWOKPJo4WPEpZE1jkw8mohu6pHH9z
+O5rLEHGl/NHsavWmhze9ALoSenB6DMisiDl2AAAAIB1obuueyEHz86CqYIOSEZr6
+8eAONYY78lbL+at9dqYOD6rKgiDsWGLdpTAN7k1X6kxOvpCKrFgW3BM6eKKsj7cp
+YHp8vANHWohrwXV5+yL7d9Wkeg97pcs5DXAFkqD6Q1KBHPfdM8V8hJoKdUODXq5y
+REhT0izOcMjkQSUV4CDr0gz9TiVylUI2tg/ZfctLJNIxKoxEREC+J3MS+xSRm4Y8
+rcOwR0bV24MkclRy94C+BHdS74/Eynr2gBpdbbe4usjttCSjSVeP9ae5ZHxloorm
+8v5rOlNhYk6IaQhtlRSRCEqGrfeTDBi6RyzmdNw+E8+EpoizKtCDbyRPLEX8iSsM
+rY5ANtrFR3QGxbx+lOeRPBM+Te9Gw0j1mKvx9POBuSx4QJ8ceTPa4xJD7k1haclX
+GsYtJLxSnrs+vMhWqGBcea2hd0ZepTeBWmv4o/SfHrgFzfH4Tfc+9EhDx9azP1uZ
+kEny90yefLues4oeBkv3PIJ9ZkN9tMImjPfiphZfYBID1zJDI6tvqW34usrlDNIg
+ptLyw1qwdyh34v6x9QcAgSOnmV0lc6mrNLt8VZqQBNTdFQFVuRN2Lof25WUCHwzc
+1kZ64jHdbu/4ebrb6RR5dzXTTBC1oYu0jeCubKjtWfu99e6hfSZ/28VUZsCA0SP9
+IT8IYlUcUlTNbpwU3ZAuHUvJitXBTatdss8mrROhRLqT7nKxL/+W+dgC4aihIh3+
+X3VarEf5XH8u2sBz9CjUiAR20eXve0C8Je+jOOs/ni6lZIZh8G7Tkm+GE9rAcPXu
+z7U2opxQuuFNGDI6b67XyBTv5Z1V45P2amMPOq68IuDOH0moT1yNLu1EkmKvGmFc
+MNBCaKWWTj6ncPpi0AtFa8wdZETNyCK5Ljh4wiSqEQb6LgUtH0XJ34ypyFUhEbr+
+Jue+5TulZh0O4ltEMXJniLkoZPiEDS2z9CDH0DAVwLc78UZHvG/DcGpljew0t16X
++xUciM5oG/9qc2m/h/sDga7Fs8WVe65iibxxB68xgXjs1P6dlVQ6ssdrqKcNz1jg
+/F/cPX+aWaPXKsFh5rJveQ5RzCCzZxGdcVC/nVJZBgIk3CuCy97LscwRhVmhqSx2
+JRXfVMHVy0t2elNnaWIR7wyhTUc2B2qZkPiWw1A7TRdbhNQove1S1St9B/3DATty
+s4XxhBBFYkiKDbcLDwXHUN3HOrjAUR+4mcSOHVLnotuHjg6KEOJLxrFrdEEvQ1CS
+OLJwmJ6ulIz9aPuhVEQKXH3qgQZFlmZyukH0x8qfRtJ5+82DQ0SbZUENcVjwDZwt
+CCBVtHJD2H4pC1KkluXrYaY+ReWwVXFBuyL+UqJp6cg8+O8ft/y8e44LA+tdojK/
+hbEuMramyuJkhGsVuR0Y925A/j+hKpefISWqGVgWJEQ1Ry0Z7cFJo3ApNjuyd+LF
+5WW0jCZZVehvvtso29qSrTq9zH6QiiJBYd40VzLA2TlXTkPFJJN/aorkRysyfHX8
++2Gl4yqC/IwTIxupLgzVQd+9g5922e8VKDRV2T2x1AbcmObr4h8FwFv3FvB3Aqvj
+kKDQk7u+dXN23+OxETQ0d1Rcdr7HrIY/QL47/v1UiSVKQKUp+6RK6iX34qOnkWBh
+oXtvFuUmN01yNXCLm+QuNbDo8ESvgpBGAzVOtUoSfIQRIbjcONQvW17PJOnRKKcx
+bpYVImf3EqZnZG9gqUaoUYed40kW0RsCH/PCYaejfnByYyrvPoHNm9a3us1NFZ3V
+g17Hy3LE5BwrutFsMHfU2fnFQBXAnOzmGGQarSUSyaLprQed2qH6hE848zn035q+
+IMeqTNA+bjroL3R6/IXmkstihLlD5FMts2uCm6pa3QKInucRhftscgO87cFqQ6uE
+sxryFRLk3Ij2M21OWfo30/MaNGWZ6qeXrss8wvrlMfloxc2ERpHXE3M9ocQhWzpK
+KMSTojQjO5CxZU6pu/UmxALziCm3+lPQwvfbwGcC9ewON+vWmxPuEsEW0hAYBH5o
+7j4zEJ8JMn6HlFEDUcf/fI1YsfV9cJScmQSux1IGh1sr4rHSk9pKXCwyf0U5ZrAA
+KTDxuA8BVJLk/3nsoyYSMOm9vNVrgCnZ0YgMfHtwa8Nrcz91VhJ028FEmhtA+bn6
+TJeODt+RYY89XOtkhS6D03y7n4DiBa2TuWevnXA0wdhOXimmgnx5tdWKDtQQbRvV
+bthbvuAwqWW+ccrcyhkI+2uvDpTO2XAvcnoOYlHqN04rMo7TlOiVmC0/MGXiLvkF
+dLWFuZ67V/l7UWCLDKG0pwweJefd2b37gls/mpuQxK7OY/k7CEjdaPLdrNxlBNke
+1VnATpDi2cflx6lm2FF6XsCyQszxifuQGRtQoiFgZrC1ipalVpeUs2LHKaQid1hx
+1e0yaqmV+zPObh7q4y7wxoTLGb41ABI2jT+bSYB64Tca3qPSU19+xouLJ9fNGEkN
+Bc1kFoW/ByYHRn6LSChwazjPhiNpuw/vbvmeeQ+NgAljVmoCwGcE56ACjw5b1OY4
+KcoyuiNTkIA9CkwgQbvdj6dyNncitBXAvW9RHlKIMlhTC1l/B6WS1FscPmdZOteL
+18CW2ndKwHcqCq/CL17z4cepThmcMKIHvI1NUde31HHFY8A5hNLrybHIYtdTzUW7
+KQDJQWzSZ40c/9/DWON/6POrhRaEnwexcTkPopk0ZxTrqPUjYvtQFkuiiHop3YBL
+Bq3fnoym94FSqg+yEjtEeDBRB3xMuZW4pFek6tp6CtaqfXrhRALPEc5ot3sEOGPf
+Oy0oRqn4bwXwWtE/xbRRu+bIwD76gwNrW06w45beWnr49xtO5WXFmoiahgGvdE9Q
++Huy5XugIJ1ycVXNbkLemIPFfRBtUCnYWb2pX+tfGLKRg2HHvXnWXYNBGaPdoJV0
+RLamQMELYM7OHbhbcIcPIMpdx6DTSjnf72oLH3oh5DZAi+XmuXbbJoWVmMm8mHTr
+995HqeB3X19drYI/1zkTgzIzXN96tYTryOWel9+EsF2bTk+0Xs02bmr3NergMUaX
+VV8+t+5RQoJhD+C7WARJ0wVZkQEsuR3OL2rHWHQaUKEtuJ/09Hq45hYFTYeolyp2
+Sjj9fEuFLaQaKppid/J80OOaffcWo0By7zfyrntEOcYY/KqU0cd9GH/P1DiS1t1m
+5f852ONOYKR9nLSJAMuQkFefrCQjb3y1cJ1bTXtGM+ECvterwbi7V3PSj4YHBvQC
+0r/NZcXOVbDF+MEwyqFpQqT0XYiATDtGRlqfyxDyG1Sz6+5S3/8Le8q78ONko4jO
+pS0BPTU9OhhY2Ny4ZLuuoyDqVHdv1acmzP0Qs9iN4H741br98YL2HJQU10nQHBvZ
+fJ7j9SXov1gtNmhu4HXTYDY5wBr+xJYy4VGaif0XXDCv0G41gdo1xH+H0tV+MlKT
+0BesyEFMGOkx4ROUXQf603XbusND1CNMXeZVLximwoc6aZhFV9ujyf1hoNbC/S7+
+VhTH3B8QuLDQjj+GtS/lg/MtcnWCgO+UGx1jctUI6efab49SRdgupcEw4NS+ABc5
+mgSAKTf/A4B7AdnPhzr38GFIpTKvygiXUxBZVJo9c4tOWJCK+ECuspUtez9HTUUg
+QR8yAewrJlseEUcuSuJi3y8gLBB3XlkPXqWN8bORtxHgOPcJxE8D7dU9zKFM3g66
+zI/5PNAp51E86d56AFqTs8jbSXSV3OajZSAeCYDnhowv8wI24+7Y9kgDiFmm0D7i
+AuoGxOtaHI6p8lFtwZ2FHGt+VLcUhPdbUNpticqYkNasp0G2I2+cT6FDLl2C6Bj0
+UpprQpskkugoNzGDhAJmTp16fyj9P4AFwhgTxOs62TUE8nUxeQs2vkftzJnYKUFi
+x9RynvKdBLsGUi+ZUOJOk9y2xgeNvFkDdNNWGa2wxI3bHQZkU6d8SNHRwUwWuVPR
+C65sTM0ky4wUS66eD49cdaH/8uFJl59yYnpfwVpLp6wLxPilSzqLkfFgMeXPTQUS
+7B2PO+6FQMKJOtbyk1OraFELz+lhpfJERpckPvjf0Wa9o6ZJ1z+6ZDPCxB/eSGbn
+iFbblggiHmhGIfuQbpZhyaBUG4y1U3jOY07OEHmOpuiFu88Z6rwPvTazq7qw9I6T
+tamkHEJtU9K8gOYLacnYBiThbpw/z5+s0v3WLqVibuH//R1jBG5PCFTQ8JyGvlr+
+xxS+PdtESXz9WyNG9fX3LPIP2mny557zIRDdXzJLnxlnlPj5d+TQBgAo9kHtkmEU
+EE42UzOu2hpObM9MTQ1v+UpZ6BFd2FyIgLdNx7/4SKRsp+yT0Kf/pM7w9HVNopFu
+zYBKDlNuyeZiZIBhNtHT3m1aKy8InWTWgYM26zv2tu18ay0XlLYfdvvlCa96+dT3
+UfOMVIMw4FqTQ4B3hXtk1Oq2fNCdWC/eyKBsGcXckpGwRDJvbTpDOapFWB4bDpMd
+UOM1acfS3BPIYNvhTmfi+nTbLBM4oMSrQL94A4xojz6IVTPH7KzKS9/f1ZrhuQ+H
+3wmJP3w2oBzczMS6949lPl0zcOd1uufi9idoSGKqvCm1XuC9AZzUw+61IfYniuIX
+xYFsh1XhTMKazeJtoXwpET5pbAwGCjWLopgwQ/tM+/te2LVuxwETN1LU/1EYm9wC
+Q/ETT/bNClrsUQo+4qyfEV2CIA5STGYggP1TXIBJ003IPNLcgYp8asBMoHkrAyOl
+0+irbL050ae9xSpc7voBFhNIAt9NaTBNxZ7AtGblsNy9pxes4Ql0oQIC8v+ehddw
+TgeqcnUAORcK0VEjQZ8OW0w4a8jRYxD8YpLyGb/xKnis1O+2oFAew7lghN03kGVr
+C6qn8HtsIpGU0tmCQVovhVdzFDO6noRYuPFS3L/WrXBoxAQXValj4fIGLhDrtbiV
+u6N8HlXpgk4PLwjLjtbcOBUDcPJoZTWgpH9A/BNKpjpO4vDOwoe2vtRxG9f6v65z
+xRGJKK3HchSeR4BUDxi2/9PaAW4MgA8jZvpBRTbyzcfMKQyg+z71ijbAuO+3ghkS
+xpaVByl9VFK2F0CppdNYeTvPdKWnUNpPHtTfxYHclHLz9xQg392KORC9kWHp7wTo
+usm6df9pwvurKtcKa/QzBofD0wAktMVPjwwxT5lWulzcjBmV7IyFyeXob0zTfys5
+6Up9z7tDwkzUegQwbgW8dIJbuA8krqW0ipZcXBGUrWyIiGi45kqybPAatiLQPB++
+3HQU5tfruBEnLmkIOwO+aQcdwrBs5cv/kgHsFfPyQgdxgIDElN+Wl8UvYKDQgnWw
+3uLz+EhywwjQJLX5cFMI8mHVu8db5LTO8ar7rrJ6XviYsLVDupGIJjNixCW+4dKE
+qm91B4X2mDLZlaEAsVSfPa8pDU4BE3G14fWb1qRd3VsOF/NpJUZ11m8Du6jskScS
+B3+xlmN6Zs4T6pE+jBr062Cmma2Ngt0B/VRAsWSU8fUlvGdrNfkWGJDz1aA6xZ2m
+LLh/x2SKNh9kP/PRcO/1skO5OaoSYiPBDfSehA+4L4mMpOTpkhl0R/VsIyvQVcEX
+mWFAXH7DqL5WB7XhIh5cwzdBkIqXVsTinlDHujD0YvzayaLDXKHPr2riTocXd4FR
+CWe/hk9r5ykGeRyYpXGwdLXh9uCv1IMQUw6KHn8VMs8fJp5Wafaf/JYgZWkPOY3c
+jlYbW3bOaaYUFNr6jbD5jl3WqAOjXRlmx4wTIcsqxzRnjudm0pHF4G6MSNTixITc
+He2QnYveHeUtZNS1nTEdlMqXzrJbbypOcZ6QTrjjLdltr5dJeAni0yKoa2SISgdh
+bNztr6pjqqDsUWUh3MYZIANLS5XRBmlKluP6gODFjc2M9JQNUnrHYTVj9raG0t4y
+6VjDf2qmIxQ+C5qBiULdufFOqg2tuo5KZeka3HHw+g9cPenXR+Hy7wYBhrcp72Qg
+5+uPTpOIdwNNNnBCUgalaaUI8k4rIkTCUImrv8ATvkRfeCn/FV8UubHnP+v3WMfM
+Q8xGQGtmUCk7HLXChvFs5JN5zSNRPMczB+tlGOAZg3ej+BNzy2cqHGtptBKq/ci8
+Qu41eOFM5FsAS4VdYmzRQXeMt2AvfU+33AWAZBqkGcapyY+X9iWwPXBixTYWYW64
+y5b3yv85ptzelY7tQQiVNtAW3xLdprjIuwjxgSoAaIjtfORnrtrEp75ZH9Ojgna8
+ad/w/1R6UL+tHHLBvTfoCFiAUjHhi3ilpG1c70d7j3YzO7431+KAkw5ssv/MgtIs
+aY/PL3X8scoCRRfKs5TTeIoqPEDq/GMSX8wY6y897DOLxLDTFuKu5T0F3lgB16i9
+0Ly9YY/t1tUetNFg7elHVoLfa1KJHlnZs0smdKRhIcnmMnVcJ3lOhyIlXP5ukW4X
+BRCMmdQJ++G73OSzlBrjg5L4uW4Ct2raN38GpE9thXWkeu0jxfq+urRwbm47ZyS3
+J/54NS2hJc+BpvR5YIwV/7Ss7IC7z4LmkWo/WS0xVxSK3DV6Kph3k5v6paHrJjx7
+8z9mva8qoBLTUD0tX3TkzBA/Scv8wbZy/6F2fOt3PGCXey8K2r4DySti44JEZNdr
+1LJraB5+bcIBponzl89oEmeBq67pQo3pJzwC9gqnv1xjtSvcVoIanxA3zc3Mvu5t
+VmQgAA4S7bnIL/Gya7tBOVzGT/tvbAYpDzDRstZ4xJFEWBJbSUz4KM/gaU+l92hb
+2VDflKjvYJvyhSZK8kserxenaya4BEB8EC4T9QFwTNPtU5pxX/Xk39RMxX0KwwwL
+nadaU6UodsZHa1SDapCz++U5DlYZi8O05rywQOvuSQ/egGBHbxbU3zk/cE8W7q+q
+VHLE5MrNa52rxHGbssEHDzv8ppS+oVpxNs/OruaMIbo78s3CoCfer/AqDjp0t3e0
+T7lMXAUtJ9CWIX9lrCUUNmr1JIria1tDN943NmC9hwioQzvnnzryH4jksKWCjb/b
+bnqcSuYe8jpfeu5M3kLiyRKFje3+hqjhR1rIQodWlWS924Hfrv+KXs+gciiwfH2L
+wHx9UOx6mDXgJKs4XJR+MDC+fJP71IsENASak1THqAb7lll+KBJgL0O9J0zkztcE
+cPe3KxYMNYO6jYnhAAAFOwQsRD3Q8/G1pwif7HRM5PLb31/GpPubq64lVIdbR2rn
++PCztxsS0wRQgYUGHtr+o1SEVQoNZpNKBnmjoV7SGCATIAz4cb4jfmbC0g8ha2+N
+HbtL0G6rU3OXLwjGB1CguZtZyuvORDCX5tizilh7m7NPv8QtyhIMlaS+sjFymCdf
+ve3js7t/Ar458vlSgWbFUgSFFsqneomW6wJg6aF3DmoO32P22gGsgE+Tu//zVxS3
+fgoKrPKF9DPKvBm7/HrKYfJFONbNIg7dA4/SZWtMYkFXA/R+37+jQWN66kZ8mpTk
+Ix2aU8uW8/M2oWNNsaN/T3N5oBiEuPEBjf3vKNul7xB7VgmbRZm5Lom81gFRc400
+9kwfeQ5SRwyaw2VtfqnREHJvmYUwLjEq+uc4IfUduAZFPslmV9+saI9Kdx0Aw1Nz
+sHSSUgqx6ZsySW3WqQt7ed16MzLX7Zks3r+skoTwCq8RppnwHQJUy/KtRUXuap+c
+FqZcPM3hPjRl2WaygKzEPd9j14+V9O/zkWk1eLwlD5Wx/Ou2vAZhP45yqzGamEpo
+HXthWnll0dMQ7G/ye0hxxbU4dx765w4bVClBY+kKcCpSR6PTpo7CdKoW8sLiqoyy
+Yo4SPZZzsfhbR/dxchJHsGrV/wwH28YTLNsB/aEqLDYxEy0TN3K5Y97Ot4LNeu3J
+7E8ES5PrZNAYFTn0OapgE5lQm05Otvuul34tVt/5SmO0UlaVjOZP038XAYnn+8BG
+eq8meCHwnd+2++UfAwuobKU9NPZ2I87/I3mPBsYGxgGQLQiuPlvNyH8yNNnLl5BN
+0+lSsdjCi8Z9tpIdAfu+EUe2GmJ8DeJvZbUBDXBL/QGqglq3kl1CewnS5vsq9+zq
+g+cAk4BQ0W/4xKFNsJA1F5ZDD34QSwfKUZgeVnRJp3S40rGz5/9Hap7YggPCFSQH
+g2SWF1/xpM+Whi8+YKVRYRYTS0n4LXgDWSh8don7/dYqUZhGXD5WleLrdX/aZekp
+19TaAGgeMREyJT+nXOszmYNJ8/HetwDhpD1wmlkN8GZoe9XRrk2A5hhSVi4PWqRb
+YUowGF93Qhi7iTfoJKJahGJN8t3aDz0WPKJFvEaOKVMH12crZWGsdUylzkSf3Ekh
+qTp5Ognq9F2aqJdkfbT6eI6Dp7gruECPnm59UTvhiSxXQZVjy9CMGdHOrCLtqtGF
+xYN4Z2CYGg1EKF9vyN/YCkJgxUNXnuWPYie3vO3kUNIa/NGID6vfkiLXhqRZtYHg
+wJRafUklWLiXjFQ+huuQo/+ptyAu/3WyP+r5b9Y4F4FqKdjYgI0XM1N9TvIUw25G
+9vRZmke3aU3jDdxZfn2ecp2QaHmkBRqp8F/nyoM4S10C8Kv48XwqHXyfxtSSY9NF
+kKH1WSI463C8kNwT7dzKdOgI2Jmgj2fY+gznDTJiQAag4h0gZrSllMjfC52x0NMd
+y8E+MxBKAsrJS0bUxp1o4pcFq0/4HCFDC4KpO4VfzP4RuVbR68TOZesWLMJ4TfAW
+QSA2gOIcPeoyXA1Syfqeuh5w012sbaVAKb8lcVWXJqAayeOj1G+azwAof88mFw8V
+VwEFcHDWK6c1cJKirz7C9XyWOBWMOYYzX3Ul9dayqwp9sc2fskgYZqDQ0Z6TUIuv
+TZ/2yQo8RkYyMl/Z3Zmp+LaPr1n+hEz07TSEHZUObE45KtCpA7DxxX/gtgyPICwZ
+Xh7zaVZlH9TlGXqD7EZDzI98hVsHtkfH0Gw3igYaFXJQrTW0OmBBobTCY2eExHRj
+nVvtPMIf1T2g7BhWk3kZha4golyooD9Q3NVlrot+xv5IG2kPXJtMtbwH6yp33K1S
+aZus3jx44Af+sSaJvK1RdI6LGGe5gaVnURxSllQjxaN7x/XsLQE8SoUFGaSfXm5b
+lgzB48lDdutYOi68jm7yKaHP8Bk/Xjm7UZ1Mpn2ZHcYf8x3+o26oRBgTqb3TAXkh
+7qpXUArBKIRb8KaLOhwQveelyMeFsen30FQGrfSs5aOeMsD0IqlDKq+XU6Wakmv9
+2s9xkzI3Urxw0JPL9sRqq8hSY8w6Ae2y6haODvOcWva68w+aUeDt3rPcIGkQb4Qg
+X8oRzP/8GpNyXhRCIvo3GKAOcr4Ls7qzOtDO166pGGf40Hft8ZT/roBVbcrZSLkO
+Ijm8/hvuOYpQncOaIfKSzmjw8nBDv45nWeJBqiviARUlJHtnyUYLhULwVObcDXQE
+sn4nwHYw6dDUtMU5uVFjsR+5VjOXbPWhspZ6AGy2csfAjsvlEaxEB2/15n5pmDOE
+oOT0lqmZ2AJHkcqtL1Fqs1Hgma5Bey9+4jXc1CN5UVR23K5XY/tv3btj9BxaQizR
+W4CX0TR/s75JX93kbhHZWvT0mRYqonx3+8TdR7IOBvHsGeVG3lGK1v0OHWxL1KsZ
+WSAlEuSJXNlWMH85X30zjb9PGcUItpMapkBpUhlSPF+9G8GH0Bv5UXyDdVjwWNzp
+6BMSdlxhgLiHn5AdtSIWs/dgleRPV9v2BAzqVC8vqWiCmYbY+HYUkBE/H9qpToEk
+HDeKoSc+PWyHCE2McxZWaIK9BqcEaqhoUOryBaeBkLR5w3EL+q/KCjYvNK3TZFUA
+OnvhzBj74Mr4A+0wfwW8769JnVeMw58orcJeOvTe7SVjGpso7ZOVAUeyRed0iN6n
+kO33HFv2zZEBPpEZd164ue7vfHpqHBAKwtFc8bpBjYLNWqTnBHniYuMv03Kwjc0j
+LN868aBfTxU3dftdPNHh//H/q4XnmTWHYYmQH+IJ9zB92GIj8quiWjI3fmxDmWMN
+H67PLOPWM+8LJ4vnfMsbEbpYpdioMd5eNckv2Ip8N6/t92HaPRyKZeRH2Nyc+inq
+H2EJVXfrdKtGtDpENDsJTrL2uOqGLNb5X6YUOuTEvoBwHx+cw57ObB5lm3cUw7P+
+Zo6/+qOrGjuKOS3NPt4cAk12UOa3bICS0Z05a+7RzFkXcPQir6J+mWxDzFVBV2AL
++wjMj92P3xFyHIQT4t2RFqZHxB/aQXAQxPSyeHUAISOywwwKBve2o2Au8JmjRdKL
+IrNFzPhJjKkPoq6ruKjzVXHYjwCDUZ7FLA1+T21WyjBpTm2iv26UNE8h2R1agE1i
+E4ybOl8LZXnUt0LAKd/MbrATX6sjC/njj7euG8i0pm+soyykUOi7SCkaHlZc6MvP
+Qj13J19Gd26xq7WRcyJ1o1jnXnYrpqDficRt44CsQXxm3rA7KsS1wWLSHysdRw5g
+O+fH1Wh8Gz+JvL8LVA+b2oRb/4zbNLgJ2nZOREAq7xM4knZlyVYG5XD5Rne2vSYO
+5YJvvGE5/eK3poRc5UoO99fX5KLL8aU4+gpBfDKVUI7FfJ/SqfXcRwwc3hs3/7ed
+nMP7JQAj6DbXq3gZ7rCsPQiTpQn6ihkytvehfhbZ33qBGmZARGCX/D/Y1qtLQskW
+GJUq8Ce2urCAj4CKtQsxUnOGIZo8DKU51PMNUvtkCTZE61GI2tIHqP8NuN+fDbcf
+obo67V2D2pX1wu4aVDlXMneCKF3q6OativUXnqq3oqVemWQgvxjIFhwCfs70YRou
+LbKo7IcohFAdb6kIaNKSZLJIhPdGC4pj8PmPUKdKSnjb8IKPxmweJAn2ti6Q2Z5x
+EwLo/uC+uYXhEtT7nJ1ifBATxHtpLatjOm6sMnfpVzJPsCPWixs/nP/cQ64GAnnJ
+mfhlKHT7ieMs4SNYIGiDIupaSFdrBr/kKS6EHa0EPA1RYwz3a8g4+fcVjOgwpiTi
+V0SH9JHL6z2trV/voYSZF1+tsekBQhwRhZcfVnJ+N81f/4k80cJE9F/UHNd4JMCE
+f9Ki62USX1F8y8fQdSV/7mSHnjnnI5nD3zFbUY7lnPE+0cyzPUSVAwewTy08CaUc
+kP/fqCx5puwIJpMr4yqb3Xt7e5QHTIbDxxnXUhrqf+WQ9w2pWD+GbSA6RHRplqFi
+jO3QRwH34yUFAkxUIsjBDGf445LAPy9u4usuoxw3f5/b95HVhn2GEtqj5AfT1vDV
+z0/ZlbVZKVjIeFjkXXGSRVNqnS7WFLZ8Yf1Y0jpG/o1H6yhrXQRdyra22sMXGozU
+5kBEiHdjqqTTHNqQh7Avw3QKOyaIHy9mDlNoJ6jzeUYOg3+ti1EAnmV4y/6OV59m
+uso0zQvZo31UPxpBucE0IIbyfzSG2FFiGhwUxo6uCFxaJZsoVea3swziNR2neNe3
+H+E+dKj/v8cf8Yz6L1NrkzEjIvOMQ2la6soRLztZ1v1lZqTFMFYNWM/F4Oj9dtiX
+5KanHFCAOxFtyXkxZ51DzJiM3kjutgBofKv1UEtoZsSqlaYgl7VVfuNI0eYiAhvO
+9LQquZ47y0L95g+ihPChpYkrEMqIEv5shjKmCp3iZPITgHO0puezZRv3x8v19Mb6
+gSStswX9Ppz9ZG4eMp8WabiuRIu/D+Em9Ce+26PCGcEkH4Ge2VQQ+2CHcZcpzpiU
+8wUqQzGNX43I8sIfNByDW+8JF+799qLQHhKH9x9zQ9CHENXSUamlX7SYSSVrtxJk
++76YVJiqDsoM0zWcyJyDg6Wcr/0VqszNFZdC3SgJcgYxc8LFLlPb14orly+aop/0
+rbErcGnCtBfQqVBmLvd/Anxv3v/BKGbqbELCEg4q+f6OCYY3PkQKMxDUhbBIzGbZ
+lCEnDsnMzbAL9qd9FLEzI8DGSbhM2b/qjRnjTk4qsllxLxg/SlS9WASnmZNgK4RB
+yvYL1Y7WUc0fPqSgfkjS4KPNgwCxpM+FJ5EI93FCR17u1GDIhI8GkNBj88Rpif3P
+grC7a4eAJY+jp7NAedOoJ91r1VKbcY1MaFcBHL25XK/nYDPUkmu7PoWPJtY3kmZ4
+xIPYQJLS2UXRcttvjvZo0+ZPlsWZUUDxAFW/90IQOcHgO3StevV795MFA8pmA3wi
+NlH4KlC75aBW5zf1vb2UP7NdeCI5FgmACwoqZqTX16aCK9mNxDHlHz7S661z+FIN
+Tea843EHLdvGVwkFfz5pO8ip05VrIz28P4ASPWzf0iys9Ev86TzyovMwOlK1AS8Z
+hVb6/Tup3jgPxgMNfuzRT9QSb2OCi/043CFvTqBveGxSTMdfefpZguAmxSv8U7GJ
+g2+BA4dydIixbuGPQlbJWWSvUfjWBDUFEPKhDvS9P1YU/AkJMfNLCC/lcYc447yI
+1m/L2NNoCtxYicHhSglVM1MB7fx/8gwrg5cgRNusl9as3y5N8QLxFeh4hkYiKZX2
+IXXLtv29fn5gJbpXR1CD+lzoKe8fWl9UHGxPuW6Mv7lyUBKn7I37zyyDC/M1MDdM
+5otfsQM2FkOXFSu6rfCa2LWCUyuuLYcwCXa4kt9pRc48eo+b9Pd0OjebDk4q7Ead
+RTvPPAXNJHR9GsYxIcaDOI/KyfjKZpnTVOFTgk5qMcRfZDTlg+2FTOcQdhrKWNXl
+/LRW4yQHN416q6xDvWK5Wu5P82I9+nU2nWebBKXrqvMvEqs/F3BmnJDC5e2HBF/i
+m2XZPM/QttDoxF/DHDhRzXmeBLS6i4F4nc4sCBCTM5prVE52cQXEgdFfL+0rRUxJ
+dVTJ0sJCxxc3nmymeHe2ur2MPXvsw9eaQkjuSJyCZJgMtYT9Jr+6q0y62x9Qiljq
+QxQ/cerfK2Ggu92CTje5aLWUa/w7rVuYHHp0cVUn+cEJlCn7podgU0KXenEF6Lqz
+heju04sJcJXTQKcT7yqMW0EJeIPbCZLrrvJ2bD+dPAAmdL2K3zbQUP+84ZbKVT96
+4BdERkqZiQIRyYqKV7S9nBh6rcpv8vCR7kSMHBD83eSLCZq81JzD76IRo/LH1vxc
+FoypT133cch0XR1E7CK036zQTMOMiwA7LrOaNYi1lvbBW1HGGqMkv4pvh3Xfhese
+toJkyd2Ev5xFmRMbVQ2ABrO+Gh0+Q6aeMoF43U+/TeytLOT6K6hPVirp8vRMsZHS
+cw/JBx8feWm5eyH6CK6euF/4RN6Z+y+1YhCBoJHgfu42Lfi49swwK8A/U8GMCNKe
+I7rnuRnp612ws1VKA+e9WStiQFMO7abvrxMdR+1CI6EVr443wwIREhUHKbSRtN66
+84BE4RdHeyPonGnikEJmsk7nrvQLIas0A9vahoPdJkHSAaSoI6dCsA3UIPCdhUxV
+8EUFu1/bDwaILdbWB9dCGEozF6Z9V0MNZTy4lOkjI5vbOgrxwUqbi5WL7Ym+53Ff
+F6XeortaeoUW5MOSt9qDZfvIiHtpbu3G9BYg0JJs1uhFmk+BCutm0rx85Q3u7r7H
+HP7VkM2fFNXLOYZsF9yQHQ4hwyPAdCjj4/Q86QatV1d3y0yyDl8dWwkGkDLWtQjX
+qpfMPEqlygAWJqClInQ8iTe66T8lTY5ZluAAB3kJrT2TjPJD33a7cU3Yi0l+jNgJ
+RUHG0SkYVYLTVYN5zysO9w9XODdt2eKo7X/a1AF7TSaD7NYm0eD7tNiZg/NIZjwS
+dLBgxZFDliMmB7vh+bSLwIVXDRtNkyG49HP/s69b3Y+xTWeJOvMiS3oEfGzxN8Ch
+P3iuGqxkNDGCtXhjiUbC4ZVSULFOQ4yMxUXI3J2fA3S7dEGxbB8qOFesdRb8AQ4q
+gbWKxQNOXf5pFzf9/Jwq3pGKoTH/7RCYrjmH6MIhiiYf1OENr8eJci8jyQWZvD/m
+6Gzo3ajKCR7zIuMYj5d8l4numvY/g1xiKhNnJ1WE+oTDAViZ0ne0WC1ua5n09F2h
+s7lNWxilFBhas2vbUqswrEKX9WlOO/BhiMweVRWC2/d0EQteg2msaFAc+aG+Db/u
+mNvQOgOH763GpuA2NvlWRPyDSZJ789q+QsH0F6QZQDnWoVV6YkQM5Q/l/MRjvP9n
+SoWGF5Cc8GePcsHrbZhrwzCoZ5VGahkpOB+HGxJ8OpqvVwqq7axSAk/PNQjiEwfG
+93zqVuVC/bBecU42zWoeaDNbJftdngkLvKDAy/Mp5pucdGo//ssg1Hyp8PBmILq+
+8PHMpaApa593F2LjOFKE7IEKuEqyF07IfwTBY82OaYaEZ4FQZOXHps6jSjESSpHR
++ZFsOA6i51nJXhnodn9az5cLHOT1kBOA2+t6basuy9paUL5iDdBTLaukXsY/R1UB
+btBEAurusZv4YOuDvjizVLd2iNvkHUfW5syILvsxZjrZDDI9ksGCY7l5iUp7d+9y
+cuD/tlQiH3ZHaYgyeYUB54ecbhRQBLVgJdD7fOu69uGgM/A7VZ6sIJccCxeHeFBL
+bVVi7Bpb0Q3hbIP1LL5ht7j2qa7iTMb3cHO1C07WcTkmeud/3bEo24WgIr75bby2
+K+ToTwgkIVCsyiw84hC9PBK+jIPXmVGnBsMX+4L26AtzK5+4GAqLMgmm1mqxFS8+
+FY7daSz656xzv9f5ksQZIry8PcLfqE8Cd7wlfEpW49mcaMARMklhnQARdSv0qf32
+pIvmnAsfhPxF+2SwOv5bkdeo4ixbC0RaYzl5Bj0xReRiF2M78eLnztNmstuoIRwX
+DbMnOk0pXGyrPVdNkq9qacuxjisLEVx5Phb76VA71sXdNzosRdKRJTqY+AFl3CMZ
+eSeTkBmGHP3nSviSJ2Hv3CMVHZw9hhHo1LW+lVtUX1fy7sAXq6wxaWqDrTakNAfn
+1o+ICHJ1DeQWwSox7bdFhRnCrt4tb5Qta+BIK/2Ehhp2HmJZazAbMrB4FrwG2DQ8
+qyX1DKRhL9mpKmEcjoEU4ikFZHr8IuOs760dJKvI5lBD48ltJqrzxFTIPek+4yQI
+YLFY1T1ZaE9E1O4wmnDGqwvjrYf2HsrAazG973/XnBgUC3jIDfbAWsS6Off7HFLb
+czxzqs3vHuyzvgen4sDad5auLtVRiYdXasc4h/1Yl54x++MpRkXEwoIw2RGYNxQY
+AXES2Jbtql2Lcjkwr3HQUJWMd0AtrHTfufvXBI+HsngYx+z9F+RamnAzph20squi
+fb1ynvlUNc+YPbdgWI2UkQaNGSe6r0I+6zWi7RNuGK1Llipj1qcaEdBrUO6XQToh
+9yQVflVU/GiXDwnMR/NljH7mBb2nZ6Rx25Na1JWWluluHr7d5FaJUQFz73T4/h7x
+DvxbODqv9r4VepJ8uhCO2O3QECg9qUxeaN7brHbKKZTP38KYAPlFvBDJ3y1C/jPr
+9ooeUKe7MR/PlyszOHvk+nqD0/QgLtsYajmOnbInk8+4kUNtpCpUMyzgouJgnmX+
+KJdUC0AU5ZnSnNZlJR7aaVNP1XwQbEZTcxyXJiwMlZNVVhQhSMr6lRSbDJGqDSIk
+/4PGoKZXaToRxdy5jqakwJwP9L1Sa5DqL2ALZtaPEVK3NEFrjkRqgfiuEEezek2I
+Y7U2dsN9lJBsK96//n5pKxz8ZRCogwD7tOVUYGc+f+YVaNa7UQ4I5cXKGfreS+TQ
+JYwXhlZ8yUUu4dfpgPfiV0R9hzombUWwo9DeTvMYwGzDrW7rkCrz1BQDPhV4ItCm
+rv1y+fjCRu4OuuWdYINrf3ejgix4O3ip7OQu+FszM9PACu7l2fOz6dcM9DZpuGLl
+6weOuuxR99FiT58MVwcCU5siUxvQZz1WfTW1UatZZ58PRqnNcNAr4ideav9GvI82
+Ppo/KTHchkBr2e2RJVFeUjz+8ElN9KEK2TMIov2qXTA1L+Uw3AdssE+0ayenOk49
+JL/jc1eeY1bMPF45kPMniW+B3zhb54XQXcOCzMmMO7aap8dszIgxhVXvNkAp3gXg
+Gx/ynKmrDgrjplddFHMQk8+7n8jqk2tvDSgjghHeh6GYjiwps0J0mh3Fs+mZY+AT
+HnclmGq9NPE2UMA9eiQzakHxO8zpEGzV5RWifKYk6yCps4MLC1nVq76lyR8VoQr0
+Zj7G4OQIWI+miZEGUiuB5pW9Jj8BqXmJZW1EY0GAC712jLrpeMoaUiCP5pL4ZpzP
+eJPBF1aBn3BxgfOU8dcTDLTQzl2VRspcvJEYdZwzgFAZOmTLgMn35CvmPsiwFyIr
+t1CzQHoAYvY/VVskkCi0VtmL8Tk5mPn+w6QM3loIRlmMqd0nzKqtdtiF0V3w7Ily
+L1OX/kG5Wz8QsyrAnT38BkDaHcv7Drvb+FSe+9j/+kzdbtlWY4ThFerbM31DO1qV
+eud0p5U+kmPg8MBQ0BjW6TfqbC1Bk7J4XsUAa3RM5381+l+FObez33/1+S4Up/sM
+Z3nEZmeeAbk+gAE01UyhbkgR9czLIjTTwv+zZRbBA0EkDC+Q+fBxX5nst6KC+jaF
+kw2DhXQQpV83vqvBt+DWIqaQEjtsZICumtBBIosl3IRflQwqV85T1M0Bo8Tr6Rhh
+xFgoFBkiO7CM9QuhvUb/xkV5gdMtnl8HgC35F8bdOdSnPdb03vXvGuzL3+0IzXcA
+3elInCScL2RqKH5deD3SuoSqhqdN/7ZlbRICOdXLFlZARGvelCEIsPPPeQVIXoye
+MVzXHCwMoWNJkGLCa2kTzf7Z5Frhr5baw3XAxKuqQhNGkliRDrFFnEWhpyJgHjlK
+rWa4Ab3Ju5RgUIA66X3Zmsd3YGtF9J44LhI29Q2/wcC7A6Qx1eRRCXoGR2LT2hSM
+V0ncIzBABPuQ+2RcwK/MZ1rFHBWQp4epMl3XPyVO5oVKduV0wnDOra0siALRXWb5
+ysMxVZYhXBtg+Dn2X6nKh/5rJqZg8/ZNM0NZICtpemSrVlHoY0/GYeES60ml7gE5
+ULQ/4Vyp1KjK8EosoXRNjiE7hQE8An9LqKuosVPJAMCBmX7bKl0XDsyTNk7zbPHO
++wHoR7Wc60DLMWv0YRIN5ZXcZEMel7dXzLyXPejrOdzS4W7KQFnOR6OXcpI7Mt2Z
+HTdAWpLiwd3GkfvWx9a0aP0HqUYLjCJh7ApoVGtrUfPcAWU+cnuodQxLr3+agy4+
+52e3ItrmYt1vlPCcfBeJnRbEGI6ux3IKRT1EO7AG7CpyaAiau+4vGdCKU2rfLueq
+S6+wI7Glp50BX7ljQtG4+wHtuaPrUc2Jsolm6W2rxCQHUxO30DbwZEXX118sXtLB
+pzkCJHTAL42tTrUbIQ0Rj2FuZR0j0ZIDFYmOa7sLCvaY0xiqdJ/dI3UeFxr3Ehcf
+zHfhpo7fTUqC7XpphA2BBb5gvxJcQY3dZlN8reRYq4g30/YPxjMrhoIwqxFLdvja
+znSou7triyM9gOehBkRW6eg/3iyidWXCfzWAH1v+oy8mjrZvnYAtQZoFweIzUeUz
+68GoM+QRZAiao6v0JK7Nwn24333+XFu6tIwf8yOsTvxQw/l1NJZQmIBg69WGo+gt
+TNkcf1S3fmbVA079cxIp5SaPveCsER94Za7FIJG7qYry4VRyAyJnFBMiGttAw6OA
+2RDDn9VpPGqxGawf8xHDia2OQOL2Sje3i+88pdAVtuzfZPQdimBzkszGwMT5wMvh
+pIaRbtA8xK/r2emsgPMmjEbCYB0lBW/Q1j2rIfjwAWVRYmHFrKRoQV9C9K4nsHbs
+921IsjdY0PIWmaYU0/ewiEP8xX1uao/lfWLciUGlpowGB9S/scqBphCZg5n5Qir6
+bSnK6FlpOuez9UH2SnJ+I44HdA7YMFvFqcsLfgkCKsVw3dQdAh2eB/jg0Gby4feN
+rIts/8G0FDxLDhYwntMtUti3gFzoTkWokE2gv+JjnDQlYg6cYY0g/h2UhHM58Zgn
+CqrC19KEy2E1mgsNhj8NJygsbk4Nr6oMxyCiKbHf8lProBMhhzj3SZB3hhRZSPBG
+7SArHB/tepUwXh0NyyTioPdtejUL0iZTZAv8M2/Z+HKkHEfxwWJK8Tz/cwd6+EON
+98jfawzErzbxkbBs5+I7vrfowy8cDGNV4sriPQFT1xKWo4d+hHfxnwc3eOQOZHdy
+OVF/e9KlIrQi9ikIVo5TYoxH+l46Wr2J5t2OKLhkGZFQBf7dMCfij5T6bMoQh40V
+ClVLkRttIYh+B7XsJZT+iN+lERoCdGNIDYRoqAfQ4bYkCBlBKbbHnl5lHDb1Sx8m
+8ASUPKVoGGSf9IHGTdfzrMNKrtxMsRvAzcsRfaR6utKupv0n+qcSK+JYN6PqH6Rn
+3aUnJWE9aoFlxK5i8DRseUIGB+XZeWP6RY61AGo=
diff --git a/src/lib-dcrypt/sample-v2.bin b/src/lib-dcrypt/sample-v2.bin
deleted file mode 100644
index 0ee99bba14d5255212d113b55bb4574d3350af9f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 13757
zcmV;uHA2cmQdv+`MMMJ!0ssI20ssI2&jtx#hDZTr0|Wss1_@w>NC9O71Ofp700;m8
z005r>0;?0!QI~D)t5j=eHBNSTf)_yDQ%iK5C|CAt_A9#G{Qv*}Ap>#&Lc0oD&VJbO
z_{WUK@6cXXXg`3X@ec5+{o<r&3}*lU05DkMk0y>5JW5#;SmeXWsD|#T9Dj2=n#&Mz
zrTo$CYW1dvH@yJ55_)iY49KjAId%X503aP`ZtI@NLG$yVs$heX5t{n(;0`s0JMvb`
z`Kx_)rVbCP%7P&5SYq9!Fb(caSL#eozL1KnSQgw9I(VY2kGCmcdVIVCM_Pz$!F752
zBKvpMq<Rl~rOP=Da0QZ}`a@EI9QWNb#e9UC3Uxz+UaoRPNK?`*&Tz=&K_wO7AnVc$
z{Z1uvl|nYQ57~XoOC-`UDvU%#K)xq)68jX9n}$5C!>~t2)!Tz4a#V8nfW8EGQtyw%
z%6j&I8eMI-xVp&gv?QZRSC93lxnz80qKfA7{%bl@VPZ~*X$WnV6p;u@hOPIL3>dmc
zEar6FJ`>M`riimD(1ULzPb@|Ji7O1PjzBir#Yc1o#k_u$=aD=UK27gN!$|d*tMT;n
zfw?SrK%X3WGuq=4L+(vsX~|a_#w{egQl7g$yvSClU|e~vp?5}Jr8j|EYxtw|pB}gc
z&GGn6_dfJUL&w&$KU<lQN%Hqho_xEWvx*)DOZPm2eP%;_w8AEg_u{4&Utkgg*D^yR
zt8b}o_`1sF4ALN`((=Pvuy-hT;{LJq2LOR1r<q+PbE&H|yL?rekOb7-6#-Sb6Lv0#
z_T^;)9}L{qMtb5g-EQyrdAi%_6nS?w(@YSxp^LPQ;I3?_?OFT1_3oj4CV$(-RA#_{
z(IfpKKL}z~98y%xZk!a|kS-ld$%@s%O{-n9&nB%Cp+vfq?sBm&|CagK0^z8kA|3u;
zby}=P`CNZ4+Q4)4DAb4qcG2bUdqBJ;@1r>DKb|h7WQJkzZqt%)h7;PraP{ubwKk%h
zP`cqw7&1C<uGh#E@8z9U<CFGkV-Gs6ydvPvA4#ZBT#YX6M3Q2!8ev>8&_ZaXmQFsW
zaQb4<3q@<p9b`n!$RfEeIC#P&su2eIE(I+gMakccsmN6!5xV{+=f34TrDh!t;#)*9
za%YISC}jAA4K1_uAji-!6~MPU@kU3yZ^LkEWsU4Kw_ca~6&#4pXdC}(b7{Ya`vZZl
z#k0kgd#+-Mym1GwF@bpO)c&27R64T9YpACU&sgC6U)()^npvaQD#2mqvTu0~QOqE-
zXAzxoP`{m0Sq1_m+$)01-pjGf5rtWysVsIS72j0B)yqqEdQ)d<ViE5Qp-o3N2Wpv+
z_?E*^J53i`gw!a#?NZe%eFy!+0XuTDh4F+CMPf*b4Yvyq1;<d`$2z#cQ6IRO#Eu<O
z=c3z(jt+_t;!DP{Yji;`Lr{`9vT&H5u9S@ZX#1g5L<(Ge>VXDDmS%FgLG;JUpGMMo
z`^|$xM4M$n4RKiT4V*0qAXT(-L)d;P3sR((<?CUlK1JoQRdGSPBK}gMY3ay3`0pRL
z{JeXP3j^z2qB6gQu`V*Urpn@EgliSK9T@j+K>k0WDwm%jC8`-%79>P9M=crc!AYZV
zDK<N@cjCq6WweYYSykw7zS}6<+LEn0z07`)iXuT_-ZWP-z}Y!hPD8~clYeT8<VPzq
ze0BW$VWs0Lg8YmVBO9qM4Anv3y@Q{2+3yu7G*#I>vD60KnC9!^9|gc$_ZIMX0;}VY
zpwN@MzIAhU-{Y|nG&FZqTz0<4tcE{8zB~T?REZ@@K&2`Bq)O@~_u`|ckziq=dv6xy
zCO1uTHE@fY<SsR^=<r0Zf{;c7HBPlk5`2UaA-LQ))Gu3J&m`&5D5o)QmK7pr_Y$UO
zWN%=pMyOGTo#ROs(HjCE^TJ`LqkeF5V=C`Hfz6xNx4O+u6`j?CUdPLF#N-?+y3uSf
zchuSW#XuFnob2WpWE!m{63L?Jtp}ajq56bRIP*F5-<rN4$Er-wK5jbbFLZkRh31mW
zVuZOv<WnuPYl54qTHOMOp63yT`)qOpyzRkiL#u?d8uAqq<lKn%Gi^>;`Zv?_8Z>2@
z>Zg~k%RIvR<uUna#m$69k=GM*J)y)QTRKW8#FL^lBRi0>WlpKP^(Mpu^N1<8`cu%t
z_uIf{0`=?;H|y4$6Ydhh7Sa$H1b%4lJ~I%X2{L|%lu-jw$NzkdSh4keaFm>x1g^(Y
z28UZK;<3_`+Dcq3GJi!mX0QM$F!8t#0aTLY|9R}ACK52|y}Z?HfGOG0hzxvtaBIVB
zb3b)f5_H?aM4B5w`MLT`myQnKkztQLT<c_oE`!s2yPtsK1+A00XRn=bG{M+TUMZ%6
ze0jCiiVoBeZ5!2Y*jv8fFsWs}amw7v83_ApuMU*X*>Ep%dJbYy>Nid+GLF-f=#`i)
zKQLwDF8Kv?wS~E!yI1*pQDBP<p|qzA9wq19*}eOMTR)ncki@ReWBEG>NZn}i-K^YY
z1lb<dS-?(^;@QXL$Ejx6QF>m$vO>)9iTjWl8&IMlU}mtjik78Tmz1+&$0?*DcUW=N
z?J{bqmHRW!ZXW96F7U>L%Nf2k01`HhKbuK_df_)3-lNh}Uw+1miznC37)cEU&14pZ
zzXv7<Mt+M(C~#{y&xRvuyASVf`JQ<XjerSbR%!ykX9VY<0*?+`)aE!T$}+kmQ;>i?
z3QQnDyWNkcayEA&v=zX;Z&4mnh%#7H3t4{$rIOTJ96o1RI@gQWz?Rx~O2Bt23a`R1
zUi0C{sZJT3Fro*%jZIP4x72aPW579t((B2w$YR%1%|*K@0LejY(r1kv|KGz{<A3P$
ztA!SXp9isVIS-<lG-nj+sP!Xa`%o53qKJAa-GEC5t>2!EruTtTst>XfJ4AReQ3rfX
zxs|x2SETCNdJ5L6eR|<U0?!f7Xt#R=IAh;CEht8*_-_U9TG2nnv{Ad}$iP1Og9B?@
zPO#&a-dcM2_Zv>-WyP9^nuY<dbWc$Dd$Q$wpdg)caaGN3Lf)8z#eEQMP$}41y{TX8
zUl_8HgJH+LdDdNnK^ddnpp|q)wx&SA3t-O99k^R?hYujiUB{r)N;%)}Y6~BFA>=kd
zi{<9IcH1U}m6*xAm~`v+-bbn6cVAy!t%5(-ITM32GhE+#wS?=)<(`+{gs@$kPEWL6
z%{Fdo_ciL^F-DhFUp}|)Q9^=Y58%641WD5cS&;!OxgE|gYR6b~8c?AvxS#a&dbs8m
z1x<&jmnwEjIQ@J}g)O8SDw<+<@_f+antk^cqd;=+H}bA~L^;M7{Hm1E$9))o&(t`Q
z*4<|1|2f#>PGF>coV1Am%aD*)pR6P!Z+x|Iom)+NMl<08zSpb4xVu+#(vOA*2J`~b
zzs+UE&Q-9*_`xvBp=m;-^j(O6OglzKTA#}h@*7mM>+VwD{|kG{yYS;=qlnI>Edf0>
zJvtaz*xa~eyRM@k>Qr}c)u$%R{SdR*jo^Ox)w=!hg7zGg6xT`692?nup5yf;=)YJk
zHfV0(b<<!rIlvnJ#FjGQQJRVU7hEu}&~7z>+BL*~htk!4GE$S!7p%xZOc?1g;S-cy
z2l~@>+q%O;)FVt?=2b5krox9hX_!S<+oQ?-VW8H+{Vx7i6vx~j5V)|=jz5OAFXe;t
zEpl~&fbWzW9b<CU2<hkAZ;w(%*e<2PFyPd_02eu$1b`_w{{w(~0ol)oI`{BlNTo8b
z$_STJ5Lr~3J#&jrSdfbNK(4ZtEqgylO+_F<A2I>#D<)eW5l1dc;$q(~AS@7fURe)b
zrH%2kk+%`xIQI#}PXq1MJ<Op@-VVCVkNG^%Dd$l<>E3z(T9dQL+evhl+~%WYARY;T
z=Z1_g^8z;G?%4K71Bh9s&_3b<>ITH?S{#n4@=<NUorN50epI&<g!fxe+HHx-n2^@2
zr$M$OZ=6q|LoQu{=os`;nrlLvB$DVTH!*{R0%lH~dVeVWKY#_o7!$<nI@vV@@^vwJ
z3pTz-?aZ0jDM4b#)N-EkodmlEQZJcM;!cy?w#EmIyjcTu(^eU+u*8kq9R_4mr+i4!
z(ZNg>xl_>#u53)rB+HBxORk;|k6d-3|MKBUm!EQCdSAg>OQ)<0#Q3F4I*XC<U@_&-
zO$8F{9gjQig+RiII@a=&Q>$oE3(x6crSe2Zmn1&;-_d5hqozsMKe}Wy!o(lmNM`4V
zR@;^cA|7Z)A^VVSmSM@DR2z)7Q+Uo}PR<Z{j;82^yU!WwybrxLv#YwW^p2CYsiYi2
zZBx>`faVKn$=C)Y;clEi&!4Q){njp}Vs7F8{T*WjZchkQ(D0mwzFPjr6uv#%L`i)8
zTO&sG_4h3D587$+=brN+5ZzxgOP?8Ml=%5~<j@8HDE2|^l3^4OPBv3BuG$(-Y|l(h
z4R85MS?Cd6*j$K!w@t^t_(-H|r|gr^r~jnR@bq;}qLFURfJzQiZpr3iWPo8d(bL{-
zS}QLIon+R5gEs3s_O|VOYb_U)wjXx;<q5BP`PBDO^Ndu3FyLB~Lx6XMdt}t=wtUc?
zSTEklpllh%+>(*7L^5w}Izu_CMOYpi4wD^F<27l=(%cisVB6tNXX5&F+bk0}pv0>{
zzjy<TXpcUKRWrxztjbH@-_@Goxetfm35h>^HlQ5b%*49)k7YhxGjQj1y658dCum4w
zs=O(+Uf{g}oYceawITK=isBc=foz9W;Y`As&EjpLd?^t=X>1Gz3N?$Om@q^8O#Ay@
z*tKrQ0TVY;)c;W!o7@6J@e@z>%?euVQ3^idte+8Gf*=l3OlBZ}{Zm|kNz+ZpJks2O
zihOFoOrUuy10$u==&NkKInk%R#VTCx`T-UbNCMwYX)sO2p1`zb<*?knrx&c@33Q<X
z0`mW!h1YOS2dZ*)067;5(NQBopAK70IBUq!V-WmelJXh9@hW(%)bF;SP#(j%V1(T_
zkY#HNs;BUKY$B1A(%FJRS}%oHa}+bWo`hJq@lxEs)~#@8!~_>rsbk^t1}+fmwYZhL
zqkJA!>4HuVF9^$y*4#K1190+aWi_Cre?a^bN~Sta;_%MGhqk`daU0kAzpit|5s4_R
z$8r>&M}Slh7`Fe@+5v71fDa>P`awlD^3BK0DGZ?dKJ|(=z_{<Xf*BIVmX!x7eN<Am
z7eJ|{(^z>s&vd1yP})x()ZfK{+>~<j_Y@%C-HJI7y^&$*?*!<&$+~s_X~O%fD%T2Y
z^fLyB!_xpHw8c-43^7lcR=Ql=j2V^ejD^YN=x<EZe=9lZN`23}L&8kddIT_T1-x{E
zTeuG-uBEh!mRwvBl&x%th-kRxO0sP58nz<PJRiQ?bQI>->$njoE@=om1HNeo9m23|
z<;(w)0qhm?@<Io3fPlo5-<FrfFJPe1f_1Ro;`8`Oa>EGFB(?c)QwZ{5)w{=A<h0K5
zs{5|8dS3XLu(d<Fk%%TUV#FoB;nIYvZ*>QS_Lwr+m7xH!RG&SsDGg2m6LGcS^_$kD
zUENy_7xQT)Ms?P21G}i~ktY%df3cQhdS=cO>XAN-8uaU6rkSmcg53fAR6wz0l=1Z?
zyk~1Q`4$+E^VOg_#hs=sxPQlFiZ&l)Kl9OW@Aa}nxjCv5Vk5x~^qzzdxG#x}q~z(6
z8FWYWY$GesRlygTVL)7d!>GPi2esiM9$dpWL6C}<R>b0-P{+D3^kV$l$)dwtq0g^s
z;!cMbcY#p}XTOF|Yv(Bjc^sIfaj<l?;r8IK)PoRH4vHRs6*A8sCZ1Ml_MiNgAZ2L}
zIgQ+oRvTM(&S|C;6x#ZYu=$Q%)~ExcT^VM_j1wWtD#tWuj^}35k;UL{j7Zet#Dv@(
z?U0>|-W}yFWYo2tF&&i3m(H?VZz@i4o{&zs<1N{3ua`-93F6ZtsB2`1N(W(V-0iQb
zW2&I+QDq_A#u*?3OG}l}25Cx`<NAQ$#f{C3^pp)!ddFckWA?U&(%v%ZSi^s6rXv(S
z3z~t6LfyIXPO1&9x{gX^=^ET|@cIv2J?Ym+;qvbW0fx6J?_?n7>yJ*8h<5``HgG~x
z2Bm4G2=Y!VB1FPaiL1ZB6TU=Wcq#uCUlh5q=RfQBSjWsm%tk<KW>6_R9JRuR@oeOi
zdCenHJjXK!>tz_=8H0DD_!D!>XDS?PX|xim{m8sR?lpMfOypYtONCuxY|%k?jJIGf
zeNVUC1%PB4q#4Gk$&Z)zC9pkkV#PKVVQ#p~miNm4Ii}p+m5%K}2$eR_7T*%xrntzv
z2=Rd`0BDHqeB@`Y+Qg^6Ss&A*f_A)V-|+ubdQiWu9CE?EH|PjhfKoBxi+H7^ZCvk1
zdyjTAJH9v9;((J5Y_k8%g3>H$kIyf4{ISXcMHkAml+$>MDm*~y{9_Vd%oyu0J?t}!
z#IVyA;;!XA1>RTz*QmYFyuD$M?bg*Ew9#Pg=|@(A-)mBd9$DG5OD1%rVIj%pGId-h
zc}|BSC0zb)k!}|S5R94B3H#x@+~l*A8smeK__=NZw`$rqe+HyaZH0BDdhH{{`o6lf
zaBglpXC${L{&+Plp(W3Oru2DWj1~X1tn7fh&w}QWYCl;mF;^6d+%<YCn0J$#`lX@k
zCOmudKW4qJDxeb6P(3YQbmYtsKS|5{!M1Y$p>}-hcRXO1doK#wz5~fCV&j5DWY=rd
zvTJA_er>`5rit^H&u9{7fvc|RLXGJsJOcI#r@vfdwJY3Kf*PL?H_gq=zV2;SWFP<z
z679LjFY&T#yFocz#!vfiYz8S0FwwHsc*K!JSQ1-FO!z3z;Au~#_h?($P~ViO?_itq
zg(gb!OCGNmr)ws-1VDTcE)(?ua7@$fQ<`yK_2l2wOvQZ)!wd_Zr&?2`D0aq2YgB`3
zkhA;cISy7Ci^H_$ys$v)?nw{cfM7>&7S!K4KX6YL?yss;a>V4y&1;>j#BrOl!3PgJ
z{HBz?p;~b^&(5yqj3K%^^3B4aC*H5{Dh@hyw|BHpxlCLIEho^HA%A78B@{Ml^(2bo
zYg<D%-ZwU2y@v>>Lp$f6I`SWg<glfJjlbJ&dYnq;9`ZV0dhSf#LgL91g^lh0hN$63
zTF63&R+VJE+kxM%|B7DEpmHd%e0__+e0@;tdYCofB&#@FlzuQUzI>DW)Qbc(1e%jn
z$EXJTmRWu%5@0Vwy(diM&esHR_qQt+3^jwgjfvp^00lb)EJQuf^YOK(2%qe9Oyu(0
z-(SY0`<tt-B~*u7M{4Kz@UyoY64L}wfrSPh+Ww<dgjEU+W|K+=d846T(ik8UAPo3%
zz9W8S!qN{RYj2GmyGzh+t5b8AF9^m5P@uV+S<36qL@<};*t3dQdz-UQzr-!d5)753
zzOpfLm?vMo?c=k%e*(Ta^7&GMX2ntjg%--EdWn|n0$}N(cMfU}-(&XL0jz*ele_=(
zR}{B?3JR?9h4eGZycxUvddgw)ML5>YA`aaHkJ4ppOkzP-1N461zoS88dg?}env~=t
z9hy_imh<yAp<_+4qkm6xd7v1CxbXpv{qHE-rSA}XRtcL$nYk{Byw(9xbB#3iOdoj;
zQb!D$!)0xLsnHN}Z<&QKE-@<l=Qttt9k>QXKFMZR->hhlN_QOq!&7sxbdpjEvFV#K
zNp04t3wwFpdNVTD?U^jzzpRpk@CvUHrkU^^0#wWLtwlxdYM-1Ird&MD;XX8F*=DkU
zti(OvW7m(B^zZYLX*GDfB@dOc{Oh*724O#ra;q_#m`Z3Jdtq96Wzo|R>~HdWNO8rr
zICmcU=MEcGDM4fD3UDe?N2AlGj>2@R7V^U4s*JK?juJhVbFuhaNB41Z5=XFV)&C3!
z+r|?t+X4NdDl9fJ6D<=ra=ByP&bNZidhN;VPXtSo>txUv6*=@ds$dhDP@7Ipw)?J^
zel1qt`ATE7QdX6W=1<dq7XgXq`@lweuO@gQ@SWeb`{f@43#e?RJv8=qBhLRLd5;Fh
z2F3x9EeNhYTg}LSGBnxCmyk`<=~A)S!i&azwvrtI`@RuJwi;r54dQQQwE+!qOZ@?=
zf?BtdT|#>a(&qar_w4F}=Kzy{P|<Jr#Gy^FkTn;ULl1rsO9#qPm>yPiNvCwU(y_DW
z|3_+`*n$JX6(k3PWR@3S@ubg|hA%!~rBPuP6H7_>EqDW2D13H_`~B7`QJ6+tK30|D
z>vezHW$7u`)Y<@O9x)LzB|oQJ>ob{yN%QgEw*cX!J#d;?4e(}Yd)3jdO@QVYQdTYx
zTBKWHN-!8-cS0Dui8tsZqFRJvP4eB^4?Pw<qD8z$jww?I*JmqbVXSpbrOrg3+({v+
zI(a$?>hxWjsF!4YwEB3CgQvJFxImAdZhcWZ;fX9)L6u|6(2N<;&a5Kss?mkTgLr3P
zm>La4C|_^L-`EO5V8ugMp5>2XC%3%q<WSNY{LzRHtKX6$*M_87wSnNklv;gBC0MwZ
zj8r~`>yV@Wskb05|8=rI>iKWhI2VCxDcIP6jTbXheNOTe!)`|Q^jVrmw`on|4cu9N
zeV%fikZ5_N1sbXFU+2n$I7?ju@T>Uod@3D$pT^XZW79>Dq4ilJIO}k{klYjP+{$$5
z2-umRk7wBW4Cf6pVn7C<;vFDnw562D-wU0w(9<2u!9Ft(N&?EsOGebjooM2h1*=c^
z93evsf~h-&U(EgyxmMBZ#Li{w7A(SeP4E^$AU1&F96jnXTn$pm`kuNTaMN9^ZKXgd
zza?>1mnNVZ$>XEcZ<@~lD1Xl;7Y`Lz0R?by)+?tqaFU|0KEm~UmN*rRIfgS|btU!I
zva1SxvCW^dNEl|I(9xcgP>ZikpZ3WLJVr(`GGE!<nW^};kFQz&giQ49G=v?M4s1?2
zD$uC|u<^xz;I<5pAS@YP9`k8dWgpb#8G3{4MnlYxe1%&FwnxX%Y&VJq8WnO-tu?ed
zU_qg@!eeKI#B^hwTkSl;AJsje>=;&)c^QSSAfjBTpg&OD)n%@We#ZVt8)*+*n@qL5
z2kR<#+^te+o2=eEc;E;Au_lSUtx<H2ix_9Qfu(0r98#84BgLb8$Mx(j0X#~D1sSBD
zUT#~K48h~cLw4&}Ixf79Zt^Li&+r*PUOBr_olK^EnH|O-^Bw-9Zm2{U6REw^0eK<r
zs#j17!6<}V@TQA891y+drO3yHvFZ2FR0gf|tmUJgGQjjAsY5ESms6#hl573i&vBD7
zH&VQC(38vd#A>U^Qe(_I0qwHt7LE?{oLcs}^ADO);O*YC+#qQXZ-gLU$`Q=}{2G&T
zUKBzi`ZpM$4syN=v%0f7(9YMcsTgPY(0A?el>e@PRc*@INVyIoIlTTG?m3E3ox_?T
z@{-PI@bYj&zm8{F;z6n_;sF&UBztGcMhk^P@Kom94Ri#uekZ_oFzL|Lw8c5OQDd<m
zxmGimZ1thCmU;kewsOb7j?3i{tV9QI_2zzQm@|Z+<n)%Qnb-nHk;<(vQEIbM;F+#L
zdoO<CHQdxAc~Mk$+^$z+`)}R5WAq$aLM+i+fS1uUf3v<xU)|(x5!qVw^qCeaqI`Gz
z#N9`-4hHe;8RbUaQHs|64jpVu)T<d;ASDvyiCo!MFn>8;eKU=}PZ`Aswv!sBKxtAL
zQaoS18^MRr8~IUugLPQ&SlsF86B2e@VSu=YpO783A{MjvV3p)gSKIak4C+)bFR5sP
znTFW-b`+2iKOfquPJtvGH;SPrK0R!Q2u+N07FKA2y#}WQYN%*X>hcArfsnL$!*L7x
zugVHGFEp*wWK{q<d*RF&`{2s>1MM(>1-$RCNu5`W!=EUv!d^P`-t8r08k;EXla&ES
zvPI`~h~B4=?e`p8_RWz2K9L!BUbwmL?|gb{91se^(OmJmL5+gVTBPR$dE#Q@FVk|c
zjm;x0-#YQ2Ur!Y`b^BdB(c%B`|Eq=PnKg%DiI5-S3HLC4*kU8{tD;&mH-2nGnPUwf
zuFovv)-&%5CyVEN%Nr58Sf$vgG2UJ^$uHQ7d^fM{_hH&S9ExS+N7&q)`YGxkVF^`t
z>vXF|v^qpII|)v*_PFYXEY|s7rW88l#J+%VA0M2<p3ZC@Wt(>t!?XToj=%b&s~S6s
zIW5gT-W&o=c2MTGY=Dx{ojGgn(ac#FaP%UtqJEidL(Ek{S6~bK2+WV&kKYk;9E20%
z-H{fiN5mi6L2wYn^s;z$03jo?!wd=r_qL;8F7TP7Mbe8RvqjAKNsOrvqOPmBsPk2E
z*pC2%QJ%#t4Sr8;R?0AGPHm#UZj>}nA=w>TfK6f(jGH=N3uSrKw?e=v-^^~X6JM(%
z3;E-Zx2_w=w5D&Yqb#IQ=(|WM8Xi_$=*!PSJ$ENxMt5$ptF@7HB6Xu!=U#Rzrl8-6
z#BJk%tU-Kc-mp6=#I?a<(jO}wM-E^+=f~A(d>cQByuS-n51ZPATmOvPG`I=cc1}b<
zD(@3Gl6GauRtDv8`9^oPy(SLjf^WQGIsM|dri5JON)Gqe*W{wh@ufKW3PF4_l~9hw
ze4o;(_1s4c9Nrr@|F@l-!}}!wBj`5Qt9Tjiu&g}@lcfpziWxGt_o03k+24AB8fHL5
zV3+(q*w(8{Ldg~wl`8Njwz{x@kARA`3o%l2h9R0f45c~L^9@q_WC=Dz>rsf>(g&#j
z4Y=Q*4Ywbmx;pJ$gW8q#!tNSWIae}wf+$_;=;p18^%tJ1x1yz9nPecp7|0eJ0)Ec)
zVHz$ivZ(BbD1=ZQZ>b1q(voDdNQC!B3yNd#`HxViN=kU!@Pd!VY#t;D_O>pN*`9F|
z0_gtWzPW|r64d*gonm|t6U2LIEvsWXZmcqQ=~psOup`!s8$X=?+(WJg0(r@q_+=<`
z`-$T$;UiceXoDi^T1Z!G2EXJfE`%Me1UwB<V+{9e$T<1;6^!UGrX=E5M2GZ|%j-R@
ztzYk<gqas#t+D9=LL3o=mmgMgemBiu|A{=&!bJ36)Ew7%B*27!(xU5S5?@h#%g4}l
zC4cT@hn_j-BbmeBF<ViN<(%<8(af_wM3n;vuum;K38fs6|KF%Ad8X_LCX*}UDx2MV
zdwY}zOoqe98P`%8>VM^s_YJ97KZb1}Iz)77mZ4&d?a)U7_v0l60!&mQ$iWO}_~Vkm
zKQC_L>n@`lH-Deo_mS0xeTEX+qvQwE*6`KOPuZ2VSt(e^cv$3Jags$-YMn0D6t;X}
z{aDgEM*fXQ>nLko1YOFuw%WrN8jRHDKtzalW2&Un9NLhFurI@O3Ogo<A1`JOQ)nls
z^La)NgMY1yQ2?H0c+38dSD$9O$~4Uj*`s|_KN>-~!89O-@_#gj*im8{92CZmt_WOO
zC7UQ!=C`v9;x!$oc-OZd;XZVz|G&o{@r?Q}Q)`nkBO>#RLup#-$`LO+S=Rk!W~9Y1
zRt;Fs#o*}ucG#EXrl%ZGfIAUw$$2qnokPr+jNVA@wg6~+tMyPzXlBH!m8Kw<wN-xO
zNYUmZ0vpcsv?{rtJIg}-<`1HT@S&xND-g<v68>z4GNuZh;$-p@fOE8_=d)!S_s7fi
z^v3#uB(1Xr{XU%iWNscZpB8Dju0)H!58)>CC%)UG!WqFNAAz3PR1o`MhjEuF&X|<*
z1u8=^jbDw(^1>f99D`f$2^a4D_M*@p5{LI6b3@RF5Y^IAsij}Em`NpTw-RLgzL->)
zst(Ew(>0vPoP&d<oUi>As?5z5mqOhr333K8bHc?gQ`^^yE0-^tqM!7wu`6(C!n7CA
zsZeGv_kRL>Z{GjGC}!$xLc$UbD*67735GX5L<%zy)P=A}%x2k?Atw&W%+0V1_NRRm
zu`?sU#!0wL*}v+I8RJe)DzaH|FBm^cRJ~XPr<s#rD}+JH_6yaH)=|wLKBS<2NYdb=
z&4U23q|b#XkqGy3LPuWi)L_Vjj|Py?WAnsmiT%%lu)AxAfF+Nkr?WtL)2JuiYt>Sl
zag9u9R{<Qoxm>U3U^CQ`Yr8&$k0#bPl4f|sgV;cl(%D7Pa@%i?_Gr`QPnN}*Q9$tk
zRloN_5IMo%J9Mpj^?Ub|1p~@v1AHPjQTQrQyXBx(=Qs7ey_7$*U3eln772h03Myu#
z*Vm?kE7^_2G36gV((A2r_)-l`=Dg!^2QAyiR|y4wK50A1sneBfBR#x7fD%1y-_k6s
z^h^BdJo2LRFgjAT0WTSaR{H%rsoppb#sdw0?9oru5^rOIi~Ts<A#YBgZ+L7{Ovhh&
z`dNbDCdDiKQ?ZGIZ-E1ca&(BXZsCtYR>@gpuTl8c1T_T^@}Um&y+2kI{0Rv$^GgUX
z<#C5N<GhI0Z_C)zXbRj|iNWDY2~{&w0qy*M@(e43mmoyjte4iT-!4t@0`V2-c!owI
zDV6pib<4K>y?%aRCAwEfP=oqh=qc|XT3=KgY)`pvjK8^ZP!gx?jr-3mgA4ODFgHx*
zi(jz=HWovd6)U=}@S51Qf>SH5Er&1(cDRz?X+_RFdXJm*_jEcpn+{GY>_(kMJI_1?
z%_MYv8pbgp#)CMI%E|c3W|`Ae;ZuT6YB9uLWHjZ2?S)L|5Ox~MSk>kHv{vIJ2RDs+
ztE@x4V!2xGPxE3u`gJy)XPX42>#FlF601KKaAus4!sYFU1YhEtW!XH>(6-R%#9zZ4
zI8n`co&>bIi-CBZ&MXKJlQWuYR8Dqr1;l~TUoY(|MNCO`RLRo9LdO?3o@}Ogcec8{
zj6Hko!`GTZNbX3Sf@GKswS@g9zq+eTy4xR6idgDH6hCq5-z#CDyWN6LH@RrFlxzGu
zty`EJdUSDBC;7n%lqvhBhhS4emwIsp=(@9o=<d^t32>FuK&KP$DvVn}33!9s36krs
z@^)-Lojd?0biIn-HqcQ2yy2G0RX=**7eq!%nTY}s$%=|sw7r}ddacTD^6-)FM2s8|
z{N3b>37WjroWt*;5u@_Q*8E%+jHypu_i@N{T^&U1BDCME&`iUOivT+=vzj%CwU+k5
zTT#Xuqa?qIZ-;f?h3g)+f@I0vguk3cnG+jT4S)u-z8W1qL#CcGfq30dzfJ6|Eadtt
zs83cZ>GJeUv60eq56K4~A9-oHdm;J=uAaDG_(a~B`!BU(5P_hP;C}8lE%><h%rGm!
zKU2Ys2-2P-y63qW>FZsvvsFq1=e=1gVn9<4?WXUq6CFqGLL;FSuZ}mv0ud4w2Pw3X
zwBEY&fJEUJM|&gaoN3~aLT0i~=dSb%A*(b4+uDYM-6lcO0i>uSr$Vp|)FAMkg-liO
zMFqQG+YbhaE!Nft*FqRdGZ&_PS3?bDJh+tUBO{yJItuZ@N}G$7i|vWN=W$;brQV{u
zT6%>R<inD;+Jj~L$cTGsZtceO79h})Y}V*SnoofW>t@oteB}-9?!Lzy{?(AppA^;0
zIfiT(+>jj(A;TlUbSUHF^gQVXtyfof%S^HkUmaTs29Pq=wFuX$m&`m$rOE&nCZMGv
zbUcYSy6Hb9O^#WX-~b1C39UVojPgU@cDr#+*o#SijMxc9LB`Q37*&GPRfBoYD-QP$
zS2#Cq+2W|}f7;Xmdrc;T?A9jH;QO@LnS=94W;_yfuwccJLzW{Z2fN|<w2Q!nR}C9Y
zlOeeDbN{oiTiuVbO=pQZ^CC-n1bl4qH^8Akc&-|(WHd2?wRmHRM#ABhQc$r@LyU~Y
zMabNpp96HebV0FfA1XLktaTRr0S+pGwTi_9PF?<K7dQR<oGRXtilH(8?GTu*Ifv-N
zA&Mp+)Zq=U$BA+;BgqAsyg%mXZ0Oyn$_XCxBI6j3mwcCr?wa;LgIr=N6K5w?g!+WT
z0a%&RceGe7ZflwJ^j)E|xlLObr4$%ivuoQ@t1zrWm-T5*JMdwM%pO%0g4_3W5er^}
zX{=~a9QmQX4ZrS~+t4}#hwrV%rr<U<`Bp^ygGrKm^V+^b!SolT89+JKp;dZfL=5E*
z<^05By#HrPg@zZ9obYFla>46um}|o@sArW%Y8fdwABP(fd^(!1R|=}_tWp9`&ov0*
z69>lkeCk%^LjAB_aZWbPY944aTP6Elo(T)Qpuo%XDdwA;bZS5T%OKQzsqpY-AiBQr
z@yw;5DQlm17h>Z$QiSY*3b;zL7f#531i@p?j%kL3XMs>;<;SMZqe?LnN|Dj|k!&~)
zqUTx3UK!|ie_GF%3moM2kQ0F0>w0agF3Z|lP`+Xf&{Hj|q+Z59M^yoC&_n|2?y;Nr
zVC#dvII~o@c8J^L9Y@yY%!n@gF=jg13^F~E!GdGCd5KDUckgm?;QzK%A|G}~X^1j;
zg#qV>oNg3Q1hrr#(EEJry7u9qGw?fAo~$63919nRcu-4iRbuQKThR^SY=iYIzG1hx
z_NlJoOvd+cbF~Xj)^Rx|dgp)Lu_)Vxpd!BcZM?QC<mgWbBq30&$}BwM5WPGSzKny{
znNg<(!x#I4_UH?9E1$R+3X3ucrq*h)6)!#&j@@Z2`sb{3zt{Pa#2F&IygkC-s80fS
zyd``}R^!>6XuuINNnxD;5p^r{sr~k(i{_jQAB6lx`(&^>{#%jPsNyVJ3q)FDIe7*>
zF-7EJ7h^l|;^)rOX0qF;AsiPCvnM)DDO_x;Jy%VVuWD(_v5qSX5nOpb7W?T?JJ!YB
zH##gu(vc-PnD_x@+#?xzCzFsFh8+FpO8Al|Vei}{6&;*Ch7su0wZ4^GR9{!}?!XtT
ztTAb7gRM5CGzaI_kBA6zbq(Ye!74HBw?%~+!mi#eZ<H--;7BX|goYY+9%5N*FdH(k
zcow_{*fcz=CG`xXVK3RKDq$RsffV8?1!Q{sBIB&@tsNw*$mUQ(<H>C%s`JED$UW&k
z<0J@Ru~^kTS!hp0)b22vaK@_(<E@AG9?HOLF}?49*PIv>3wX#4_P|=ix;gjz98%kJ
zJaeke?;h;4z6Yn`z}k0~t}fM4iHBEe$2f=mSeKqL`{OA_Ma05_Fxe59Hxw8FaT3^;
z?W$diayc-sanMkejCVjStaRVG`_}}Ihq8DW$L#$V<XW0=Go~H1va6zfy>g!UR5j0-
zJ-1+3jg*lFjTtApuR=cSHKOekZWyggmMUY`ry3E^Yf$c&K{_G#Bo%&DRQza{4++di
z^JR>F<^{c{XQXl4lUmf3mX_&m9=_e=R*6vobMJKc{vPoT{98CWulBwbdXjv)5RTaG
z&=4p+sZ3sI-rKBp$|;o3-@=#x`9-`C$=@wP{xj?LiXKp>yD=Znmn$<kd*u3hgVXdN
zF54JtIgXvOCzH>(kwa~yDpWHp;G*JSo@M?hmsATt6y=%HoYrL}9@=SBPt|-7Y(`Ua
z9G50643(2rRumyf%KDWQn+%bv4I(7}gT|nyS7|yC#oW1$rli1}5A?lKYmn+MU<+o}
zj}cP0G(l^QL~4Qft`JAFdQFI9wKjIceUy-FE8f5UerYQl{ACcRg8=)q<y2s2K7ZyF
zXx6(?4hZGN${G6JOXSccj2DJhe91*F;n(Sa_u^MXeTO<GZAGx7(B4k-7{F}9t#0d(
zD)ZD713ndaBG9I;{c`#E!ba{6y5*f<gKK|xqk=4WJ9w$=<SzJIGc(h`3hw3E^RwyK
z4D>c>xMJn&2adYzQTNeePoE4|2Le-@B2ydCXFXPZHMLQzS!bUQMybtk&@196UTXhF
zypJ|Mnm;Kq+=f7F+3k@fQC?C!{_sgn^q~sbGYF#ns$DQOFXb@Y2W+rUv}-4)I!-+#
zzvFXPo?}+bJYG4F^CyXKf!{b==Y`N+!-CAoj61fPr^jr}h%tp#?>0aw-UZ+rAM%{3
zs}2g|rdM4Qa}blyyPwGFlWT7cC?kRq-iM)>jw~s&LUfuP#k1*|W8f1WcO{r=y)^MQ
zP{2KUBr|G3@jJ}v5Ny@u6{38mB<molvx5r@S=Fn)rO6)^p$hb7KE~kW2w0D%iIE0U
zD}m;fy(T{asd<TIZA4>1fD65LjJoM~${JE2kLHs2W}MG>lff5OfuC@3f%BB{*Aoo1
z(9T_zM#@~gkr;KHGk{PTI%La$$@k<d=03=<7a}XSP_saK0Alt(Ra+#GD704Di}5)*
znEC$0qzv9#2u4|qsof{cs;zd|h0$H`?1^$OQ<wfhxm!OFvns%yJ^Thh+8xXL4!hg<
zRG$0T|N2bbZrN62gy9wH+cSMbJ6e@`=X9r)K9XbL@W4>e7}n`G>TE4Rld^bT#Q<w`
zOy_?!`d@`Px3k}W_4zIor~3?NdBkRCo&mW&fB`hsOrdT_5%tW=A~e&&|FdNl!2>}g
z3@?!R@Nr+6?6;zV`Zk4=4TFVr5T#!?zN^8v;MO9hkP<s=WPq-k&_N=LCESExl?*CZ
z&QsLQ0i(q0=@?<eSSS=3B0I2*^$VfBM*qe|d4ba{o?i!mE%_J5-8s~!J=XNz_3s+&
z%irw?&36FZ=}4R;oG)Z*D1KddJ<_^_s)nac|F&gq5&}8Z%NABZL~GuZAqcSZ&v^w%
zUW}eGT-O{d454F5kYd7XX%o%<+2mT`ua?@wb-=`{szMV+l30-ru|=Fkp{F8X9yv;_
zX1D>p$-9(bP=Gq=ec77FcVKHp^qx2_5;pY>zrnz}1Eew4<WUKF21jDk+7ygeN!%ka
zKm_}c`(#|eugqs!#T*ror-!LBUDrP)PUeM5cI9-!aL%o*EQkWpU1s^p!!cEsAzT|^
z_&N4psmh1`YbK^(^Y%?MLs=jzX?kR<R#E6<PsU;466;B&?g2Sav_Iiosnn>-@JcM9
zbWM&SJB0x}0)I=WtEjP4$pFBCnSR?UT^A0_lQvHCZ1K+f0q94yoa;c#F>CZ;5)I{*
z++;%@m$z5Uyq7)b>p9%g;cm)6S<XkJmvWLjGToUSH$YmF;=$d<k^9!i*0gB-2dPF2
zj3Qy|3TRYoYf<yu0cAdNd#H5`ORs;LgDyVjXSX8S=3?D%l<=H<7m1w~#2AjQ$8riq
zJw!XO2J9+wXb76S?k^e8ic@OeF6XLCudpMrrKg<%U%6vK(YX5o?YX1tQO$|6iDv0-
ztHdM+Qxmt)Ht=Lc*VkVxUedv*IRYefz%PxhPPH2$4H1uFZe<-K(UJodiH>W#3kvp_
z(-^9BpWP#M9v2$-5*HuLcj2av-%Uz_?Rsg14S@x|V80SvL5<yJQ+%!DSgVLP)AkR>
zGb@IIFsl(ucKF)Pbf~+#Yl|a2fajqGL{{nOKi({&b!EbTHGm&m{-ZA@j<#=|fGt6q
z1;OGoQROr1!KgFj5o8FOqpS2JuFb-IxZi#LTwA)dj34tOtWNw;!})bImQa|0VC&U}
zqv$P6*&KgVw|-{T15W*O5-H^-kG<fm5g&MEuEijcyQzxu;Z$-1B4-p6A{yI3!=r%N
z5W}C<X*_DN8LS`k5yOeCjzHq}N;kKQ?>wc@6}If(Wb_@1U~`hp#=yk+z{}yJhLLX2
zJjAc-+3Bo+^CpZ&!eAXG1#i&SJ*y%3@Bw8}VqwLsq-a53LiDaDuy*YCZAh{=SkUqo
znWhxe_ppdV{Kb84YLDf8V%&*ArKXGq2h_i@%7La3nS+`6LMr-gDaz<sX*%b#^+EPZ
za(*L@2XqeDFk8i`%L{%90xHFD-P9cd9i9jH;Lv9B;rETKi){bFv=lr`4i+$;(=Afi
zw}4#ePDQAYO`yNxW1KW4Vh)^PjUfIVl!S9R@t7wHs>0XOgv((ynhOnvKMf}+EN)H>
zuc{2kAfhR;-||!Ipc5g7IQL1AcZL*MNbpAOAS)an?Ru3kUL6g~B;ug=ZF)5e(k4@6
z3;Z*0+4yp#97plNVoLEm|8oa=_(P5N$lq%W#IH8-k+5v%;yb>#=)*4@3}aQ|%Hll%
zQ`ZufqlbQkck!PGH+bX@WOs5oQGa{Vr6RN<_9+Nfj#FZcNBUkmTD^(p-Hs@@WEqiA
z1^(SIC*qHk`fSP&hm92qRZEc@Z6Sz$2es@al>Ug{r4bqebYn;jglMP-(BZZu2pK^s
zw#S}cWgIs3OCKii1e83bXc%Ol^nu1r*Ym8yO0L{Yu^Yh6%MpE~db-lCru`@SrxGjT
nSU02UAEam9r6(m}J!*kv#I9oSG;Dc71_$NYd1LxTj<o=4qPg?M

diff --git a/src/lib-dcrypt/test-stream.c b/src/lib-dcrypt/test-stream.c
index e4264d5a57e..1058b30b7ba 100644
--- a/src/lib-dcrypt/test-stream.c
+++ b/src/lib-dcrypt/test-stream.c
@@ -7,6 +7,7 @@
 #include "istream.h"
 #include "istream-decrypt.h"
 #include "istream-hash.h"
+#include "istream-base64.h"
 #include "iostream-temp.h"
 #include "randgen.h"
 #include "hash-method.h"
@@ -56,20 +57,22 @@ void test_static_v1_input(void)
 
 	test_begin("test_static_v1_input");
 
-	struct istream *is_1 = i_stream_create_file("sample-v1.bin", IO_BLOCK_SIZE);
-	struct istream *is_2 = i_stream_create_decrypt(is_1, test_v1_kp.priv);
+	struct istream *is_1 = i_stream_create_file("sample-v1.asc", IO_BLOCK_SIZE);
+	struct istream *is_2 = i_stream_create_base64_decoder(is_1);
 	i_stream_unref(&is_1);
-	struct istream *is_3 = i_stream_create_hash(is_2, hash, hash_ctx);
+	struct istream *is_3 = i_stream_create_decrypt(is_2, test_v1_kp.priv);
 	i_stream_unref(&is_2);
+	struct istream *is_4 = i_stream_create_hash(is_3, hash, hash_ctx);
+	i_stream_unref(&is_3);
 
-	while((siz = i_stream_read(is_3))>0) { i_stream_skip(is_3, siz); }
+	while((siz = i_stream_read(is_4))>0) { i_stream_skip(is_4, siz); }
 
-	if (is_3->stream_errno != 0)
-		i_debug("error: %s", i_stream_get_error(is_3));
+	if (is_4->stream_errno != 0)
+		i_debug("error: %s", i_stream_get_error(is_4));
 
-	test_assert(is_3->stream_errno == 0);
+	test_assert(is_4->stream_errno == 0);
 
-	i_stream_unref(&is_3);
+	i_stream_unref(&is_4);
 
 	hash->result(hash_ctx, hash_dgst);
 
@@ -88,20 +91,22 @@ void test_static_v2_input(void)
 	unsigned char hash_dgst[hash->digest_size];
 	hash->init(hash_ctx);
 
-	struct istream *is_1 = i_stream_create_file("sample-v2.bin", IO_BLOCK_SIZE);
-	struct istream *is_2 = i_stream_create_decrypt(is_1, test_v2_kp.priv);
+	struct istream *is_1 = i_stream_create_file("sample-v2.asc", IO_BLOCK_SIZE);
+	struct istream *is_2 = i_stream_create_base64_decoder(is_1);
 	i_stream_unref(&is_1);
-	struct istream *is_3 = i_stream_create_hash(is_2, hash, hash_ctx);
+	struct istream *is_3 = i_stream_create_decrypt(is_2, test_v2_kp.priv);
 	i_stream_unref(&is_2);
+	struct istream *is_4 = i_stream_create_hash(is_3, hash, hash_ctx);
+	i_stream_unref(&is_3);
 
-	while((amt = i_stream_read(is_3))>0) { i_stream_skip(is_3, amt); }
+	while((amt = i_stream_read(is_4))>0) { i_stream_skip(is_4, amt); }
 
-        if (is_3->stream_errno != 0)
-                i_debug("error: %s", i_stream_get_error(is_3));
+        if (is_4->stream_errno != 0)
+                i_debug("error: %s", i_stream_get_error(is_4));
 
-	test_assert(is_3->stream_errno == 0);
+	test_assert(is_4->stream_errno == 0);
 
-	i_stream_unref(&is_3);
+	i_stream_unref(&is_4);
 
 	hash->result(hash_ctx, hash_dgst);
 

From 9b1bd5d406b1d05ba0c4e4babfbf8eb88c2cb7fa Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 31 May 2016 17:06:29 +0300
Subject: [PATCH 198/450] lib-fs: Don't duplicate backend name in fs_init()
 errors.

fs_init() already adds the "backend: " prefix to the error_r, so the callers
shouldn't do it again.
---
 src/lib-fs/fs-metawrap.c   | 2 +-
 src/lib-fs/fs-randomfail.c | 2 +-
 src/lib-fs/fs-sis-queue.c  | 2 +-
 src/lib-fs/fs-sis.c        | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/lib-fs/fs-metawrap.c b/src/lib-fs/fs-metawrap.c
index ceb2b84b41d..1102c21c336 100644
--- a/src/lib-fs/fs-metawrap.c
+++ b/src/lib-fs/fs-metawrap.c
@@ -70,7 +70,7 @@ fs_metawrap_init(struct fs *_fs, const char *args, const
 		parent_args++;
 	}
 	if (fs_init(parent_name, parent_args, set, &_fs->parent, &error) < 0) {
-		fs_set_error(_fs, "%s: %s", parent_name, error);
+		fs_set_error(_fs, "%s", error);
 		return -1;
 	}
 	if ((fs_get_properties(_fs->parent) & FS_PROPERTY_METADATA) == 0)
diff --git a/src/lib-fs/fs-randomfail.c b/src/lib-fs/fs-randomfail.c
index c3132a53b0f..4cff6671f1c 100644
--- a/src/lib-fs/fs-randomfail.c
+++ b/src/lib-fs/fs-randomfail.c
@@ -188,7 +188,7 @@ fs_randomfail_init(struct fs *_fs, const char *args,
 		parent_args++;
 	}
 	if (fs_init(parent_name, parent_args, set, &_fs->parent, &error) < 0) {
-		fs_set_error(_fs, "%s: %s", parent_name, error);
+		fs_set_error(_fs, "%s", error);
 		return -1;
 	}
 	return 0;
diff --git a/src/lib-fs/fs-sis-queue.c b/src/lib-fs/fs-sis-queue.c
index 8259f0f0400..9b330229d0c 100644
--- a/src/lib-fs/fs-sis-queue.c
+++ b/src/lib-fs/fs-sis-queue.c
@@ -65,7 +65,7 @@ fs_sis_queue_init(struct fs *_fs, const char *args,
 	else
 		parent_name = t_strdup_until(parent_name, parent_args++);
 	if (fs_init(parent_name, parent_args, set, &fs->super, &error) < 0) {
-		fs_set_error(_fs, "%s: %s", parent_name, error);
+		fs_set_error(_fs, "%s", error);
 		return -1;
 	}
 	return 0;
diff --git a/src/lib-fs/fs-sis.c b/src/lib-fs/fs-sis.c
index bee09e7a96a..1563c29592a 100644
--- a/src/lib-fs/fs-sis.c
+++ b/src/lib-fs/fs-sis.c
@@ -71,7 +71,7 @@ fs_sis_init(struct fs *_fs, const char *args, const struct fs_settings *set)
 		parent_args++;
 	}
 	if (fs_init(parent_name, parent_args, set, &fs->super, &error) < 0) {
-		fs_set_error(_fs, "%s: %s", parent_name, error);
+		fs_set_error(_fs, "%s", error);
 		return -1;
 	}
 	props = fs_get_properties(fs->super);

From 24a4e2537a9cfc6194906ef784202c26c9dbc571 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 31 May 2016 17:16:17 +0300
Subject: [PATCH 199/450] dict-client: Fixed idle_msecs setting.

---
 src/lib-dict/dict-client.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib-dict/dict-client.c b/src/lib-dict/dict-client.c
index e4a4a4435a5..b4ceb038f2d 100644
--- a/src/lib-dict/dict-client.c
+++ b/src/lib-dict/dict-client.c
@@ -515,12 +515,12 @@ client_dict_init(struct dict *driver, const char *uri,
 
 	/* uri = [idle_msecs=<n>:] [<path>] ":" <uri> */
 	if (strncmp(uri, "idle_msecs=", 11) == 0) {
-		p = strchr(uri+14, ':');
+		p = strchr(uri+11, ':');
 		if (p == NULL) {
 			*error_r = t_strdup_printf("Invalid URI: %s", uri);
 			return -1;
 		}
-		if (str_to_uint(t_strdup_until(uri+14, p), &idle_msecs) < 0) {
+		if (str_to_uint(t_strdup_until(uri+11, p), &idle_msecs) < 0) {
 			*error_r = "Invalid idle_msecs";
 			return -1;
 		}

From 4ff22b5831ea240e8ca23d01eb796b971ac90376 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 1 Jun 2016 00:19:41 +0300
Subject: [PATCH 200/450] lib-dcrypt: Fixed dcrypt_openssl_generate_keypair()

---
 src/lib-dcrypt/dcrypt-openssl.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index d51233fa6cc..b8578769b46 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -699,7 +699,7 @@ bool dcrypt_openssl_generate_keypair(struct dcrypt_keypair *pair_r, enum dcrypt_
 {
 	EVP_PKEY *pkey = NULL;
 	if (kind == DCRYPT_KEY_RSA) {
-		if (dcrypt_openssl_generate_rsa_key(bits, &pkey, error_r) == 0) {
+		if (dcrypt_openssl_generate_rsa_key(bits, &pkey, error_r)) {
 			pair_r->priv = (struct dcrypt_private_key*)pkey;
 			return dcrypt_openssl_private_to_public_key(pair_r->priv, &(pair_r->pub), error_r);
 		} else return dcrypt_openssl_error(error_r);
@@ -710,7 +710,7 @@ bool dcrypt_openssl_generate_keypair(struct dcrypt_keypair *pair_r, enum dcrypt_
 				*error_r = t_strdup_printf("Unknown EC curve %s", curve);
 			return FALSE;
 		}
-		if (dcrypt_openssl_generate_ec_key(nid, &pkey, error_r) == 0) {
+		if (dcrypt_openssl_generate_ec_key(nid, &pkey, error_r)) {
 			pair_r->priv = (struct dcrypt_private_key*)pkey;
 			return dcrypt_openssl_private_to_public_key(pair_r->priv, &(pair_r->pub), error_r);
 		} else return dcrypt_openssl_error(error_r);

From eabf9658a9db59e60cf364f44a33f5b32ef59cf5 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 1 Jun 2016 02:01:10 +0300
Subject: [PATCH 201/450] lib-dcrypt: Fixed check program dependencies

---
 src/lib-dcrypt/Makefile.am | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-dcrypt/Makefile.am b/src/lib-dcrypt/Makefile.am
index 5f24f3ea518..42fe80848a5 100644
--- a/src/lib-dcrypt/Makefile.am
+++ b/src/lib-dcrypt/Makefile.am
@@ -36,7 +36,7 @@ EXTRA_DIST = \
 	sample-v1.asc \
 	sample-v2.asc
 
-check_PROGRAMS = test-crypto test-stream
+noinst_PROGRAMS = test-crypto test-stream
 
 check: check-am check-test
 

From 936f81cf71b86e36db7be50ba1b937a5e6bcdf38 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Wed, 1 Jun 2016 08:43:23 +0300
Subject: [PATCH 202/450] doveadm: Do not use INT64 with mail cmds yet

---
 src/doveadm/doveadm-mail-index.c   |  2 +-
 src/doveadm/doveadm-mail-mailbox.c | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/doveadm/doveadm-mail-index.c b/src/doveadm/doveadm-mail-index.c
index 1667e122703..b0e1207a819 100644
--- a/src/doveadm/doveadm-mail-index.c
+++ b/src/doveadm/doveadm-mail-index.c
@@ -281,7 +281,7 @@ struct doveadm_cmd_ver2 doveadm_cmd_index_ver2 = {
 DOVEADM_CMD_PARAMS_START
 DOVEADM_CMD_MAIL_COMMON
 DOVEADM_CMD_PARAM('q',"queue",CMD_PARAM_BOOL,0)
-DOVEADM_CMD_PARAM('n',"max-recent",CMD_PARAM_INT64,0)
+DOVEADM_CMD_PARAM('n',"max-recent",CMD_PARAM_STR,0)
 DOVEADM_CMD_PARAM('\0',"mailbox-mask",CMD_PARAM_STR,CMD_PARAM_FLAG_POSITIONAL)
 DOVEADM_CMD_PARAMS_END
 };
diff --git a/src/doveadm/doveadm-mail-mailbox.c b/src/doveadm/doveadm-mail-mailbox.c
index d99e45effd5..411f37556df 100644
--- a/src/doveadm/doveadm-mail-mailbox.c
+++ b/src/doveadm/doveadm-mail-mailbox.c
@@ -744,11 +744,11 @@ struct doveadm_cmd_ver2 doveadm_cmd_mailbox_update_ver2 = {
 DOVEADM_CMD_PARAMS_START
 DOVEADM_CMD_MAIL_COMMON
 DOVEADM_CMD_PARAM('g', "mailbox-guid", CMD_PARAM_STR, 0)
-DOVEADM_CMD_PARAM('V', "uid-validity", CMD_PARAM_INT64, 0)
-DOVEADM_CMD_PARAM('N', "min-next-uid", CMD_PARAM_INT64, 0)
-DOVEADM_CMD_PARAM('R', "min-first-recent-uid", CMD_PARAM_INT64, 0)
-DOVEADM_CMD_PARAM('H', "min-highest-modseq", CMD_PARAM_INT64, 0)
-DOVEADM_CMD_PARAM('P', "min-highest-pvt-modseq", CMD_PARAM_INT64, 0)
+DOVEADM_CMD_PARAM('V', "uid-validity", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('N', "min-next-uid", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('R', "min-first-recent-uid", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('H', "min-highest-modseq", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('P', "min-highest-pvt-modseq", CMD_PARAM_STR, 0)
 DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
 DOVEADM_CMD_PARAMS_END
 };

From dc43757aefdcd4432eda4d75f6f37577e05d2a21 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 31 May 2016 22:19:37 +0300
Subject: [PATCH 203/450] lib: Fixed max_buffer_size handling in istream-chain

The default max_buffer_size=256 was wrong in all situations.

We're now assuming that the underlying istreams' max_buffer_size is always
correct. While gluing together two streams we're now allocating enough
memory to hold all of the wanted data (instead of assert-crashing as could
have happened earlier). This means that the max memory usage is actually
the two streams' max_buffer_size summed together. Ideally this would be
fixed to limit the max_buffer_size to maximum of the two, but that would
require further changes.
---
 src/lib/istream-chain.c | 38 +++++++++++++-------------------------
 1 file changed, 13 insertions(+), 25 deletions(-)

diff --git a/src/lib/istream-chain.c b/src/lib/istream-chain.c
index 27fd43778a9..89c1bbc59cc 100644
--- a/src/lib/istream-chain.c
+++ b/src/lib/istream-chain.c
@@ -24,6 +24,7 @@ struct chain_istream {
 	struct istream_private istream;
 
 	size_t prev_stream_left, prev_skip;
+	bool have_explicit_max_buffer_size;
 	
 	struct istream_chain chain;
 };
@@ -45,10 +46,9 @@ i_stream_chain_append_internal(struct istream_chain *chain,
 		i_stream_ref(stream);	
 
 	if (chain->head == NULL && stream != NULL) {
-		if (chain->stream->istream.max_buffer_size == 0) {
-			chain->stream->istream.max_buffer_size =
-				stream->real_stream->max_buffer_size;
-		} else {
+		struct chain_istream *cstream = (struct chain_istream *)stream;
+
+		if (cstream->have_explicit_max_buffer_size) {
 			i_stream_set_max_buffer_size(stream,
 				chain->stream->istream.max_buffer_size);
 		}
@@ -77,6 +77,7 @@ i_stream_chain_set_max_buffer_size(struct iostream_private *stream,
 	struct chain_istream *cstream = (struct chain_istream *)stream;
 	struct istream_chain_link *link = cstream->chain.head;
 
+	cstream->have_explicit_max_buffer_size = TRUE;
 	cstream->istream.max_buffer_size = max_size;
 	while (link != NULL) {
 		if (link->stream != NULL)
@@ -106,7 +107,7 @@ static void i_stream_chain_read_next(struct chain_istream *cstream)
 	struct istream_chain_link *link = cstream->chain.head;
 	struct istream *prev_input;
 	const unsigned char *data;
-	size_t data_size, size, cur_data_pos;
+	size_t data_size, cur_data_pos;
 
 	i_assert(link != NULL && link->stream != NULL);
 	i_assert(link->stream->eof);
@@ -137,17 +138,12 @@ static void i_stream_chain_read_next(struct chain_istream *cstream)
 		cstream->prev_stream_left = 0;
 	}
 
-	/* we already verified that the data size is less than the
-	   maximum buffer size */
 	if (data_size > 0) {
-		if (!i_stream_try_alloc(&cstream->istream, data_size, &size))
-			i_unreached();
-		i_assert(size >= data_size);
+		memcpy(i_stream_alloc(&cstream->istream, data_size),
+		       data, data_size);
+		cstream->istream.pos += data_size;
+		cstream->prev_stream_left += data_size;
 	}
-	memcpy(cstream->istream.w_buffer + cstream->istream.pos,
-	       data, data_size);
-	cstream->istream.pos += data_size;
-	cstream->prev_stream_left += data_size;
 
 	i_stream_skip(prev_input, i_stream_get_data_size(prev_input));
 	i_stream_unref(&prev_input);
@@ -190,7 +186,7 @@ static ssize_t i_stream_chain_read(struct istream_private *stream)
 	struct chain_istream *cstream = (struct chain_istream *)stream;
 	struct istream_chain_link *link = cstream->chain.head;
 	const unsigned char *data;
-	size_t size, data_size, cur_data_pos, new_pos;
+	size_t data_size, cur_data_pos, new_pos;
 	size_t new_bytes_count;
 	ssize_t ret;
 
@@ -251,16 +247,9 @@ static ssize_t i_stream_chain_read(struct istream_private *stream)
 		   new data with it. */
 		i_assert(data_size > cur_data_pos);
 		new_bytes_count = data_size - cur_data_pos;
-		if (!i_stream_try_alloc(stream, new_bytes_count, &size)) {
-			stream->buffer = stream->w_buffer;
-			return -2;
-		}
-		stream->buffer = stream->w_buffer;
-
-		if (new_bytes_count > size)
-			new_bytes_count = size;
-		memcpy(stream->w_buffer + stream->pos,
+		memcpy(i_stream_alloc(stream, new_bytes_count),
 		       data + cur_data_pos, new_bytes_count);
+		stream->buffer = stream->w_buffer;
 		new_pos = stream->pos + new_bytes_count;
 	}
 
@@ -295,7 +284,6 @@ struct istream *i_stream_create_chain(struct istream_chain **chain_r)
 
 	cstream = i_new(struct chain_istream, 1);
 	cstream->chain.stream = cstream;
-	cstream->istream.max_buffer_size = 256;
 
 	cstream->istream.iostream.close = i_stream_chain_close;
 	cstream->istream.iostream.destroy = i_stream_chain_destroy;

From 9e7b4d66018c276bfe4feed3dd81244f198f0d91 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 31 May 2016 22:33:32 +0300
Subject: [PATCH 204/450] fs-metawrap: Fixed buffer size while reading metadata
 header.

It's not enough to have the buffer size set to "large enough" at the time of
the stream creation, because i_stream_set_max_buffer_size() could be called
afterwards.
---
 src/lib-fs/fs-metawrap.c      | 5 +----
 src/lib-fs/istream-metawrap.c | 7 +++++++
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/lib-fs/fs-metawrap.c b/src/lib-fs/fs-metawrap.c
index 1102c21c336..c2fd86677b1 100644
--- a/src/lib-fs/fs-metawrap.c
+++ b/src/lib-fs/fs-metawrap.c
@@ -13,8 +13,6 @@
 #include "iostream-temp.h"
 #include "fs-api-private.h"
 
-#define MAX_METADATA_LINE_LEN 8192
-
 struct metawrap_fs {
 	struct fs fs;
 	bool wrap_metadata;
@@ -283,8 +281,7 @@ fs_metawrap_read_stream(struct fs_file *_file, size_t max_buffer_size)
 		return file->input;
 	}
 
-	input = fs_read_stream(file->super_read,
-			       I_MAX(max_buffer_size, MAX_METADATA_LINE_LEN));
+	input = fs_read_stream(file->super_read, max_buffer_size);
 	file->input = i_stream_create_metawrap(input, fs_metawrap_callback, file);
 	i_stream_unref(&input);
 	i_stream_ref(file->input);
diff --git a/src/lib-fs/istream-metawrap.c b/src/lib-fs/istream-metawrap.c
index edba22f124d..5f4528136e4 100644
--- a/src/lib-fs/istream-metawrap.c
+++ b/src/lib-fs/istream-metawrap.c
@@ -4,6 +4,8 @@
 #include "istream-private.h"
 #include "istream-metawrap.h"
 
+#define METAWRAP_MAX_METADATA_LINE_LEN 8192
+
 struct metawrap_istream {
 	struct istream_private istream;
 	metawrap_callback_t *callback;
@@ -60,7 +62,12 @@ static ssize_t i_stream_metawrap_read(struct istream_private *stream)
 		      stream->istream.v_offset);
 
 	if (mstream->in_metadata) {
+		size_t prev_max_size = i_stream_get_max_buffer_size(stream->parent);
+
+		i_stream_set_max_buffer_size(stream->parent, METAWRAP_MAX_METADATA_LINE_LEN);
 		ret = metadata_header_read(mstream);
+		i_stream_set_max_buffer_size(stream->parent, prev_max_size);
+
 		i_assert(stream->istream.v_offset == 0);
 		mstream->start_offset = stream->parent->v_offset;
 		if (ret <= 0)

From 3dafbab91728c044e58f18d529a8fa80546b2fb8 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 1 Jun 2016 13:30:22 +0300
Subject: [PATCH 205/450] lib: Fix to previous istream-chain max_buffer_size
 handling

We were casting the wrong stream.
---
 src/lib/istream-chain.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/istream-chain.c b/src/lib/istream-chain.c
index 89c1bbc59cc..ddf0f566c82 100644
--- a/src/lib/istream-chain.c
+++ b/src/lib/istream-chain.c
@@ -46,7 +46,7 @@ i_stream_chain_append_internal(struct istream_chain *chain,
 		i_stream_ref(stream);	
 
 	if (chain->head == NULL && stream != NULL) {
-		struct chain_istream *cstream = (struct chain_istream *)stream;
+		struct chain_istream *cstream = (struct chain_istream *)chain->stream;
 
 		if (cstream->have_explicit_max_buffer_size) {
 			i_stream_set_max_buffer_size(stream,

From 5b6559c3d7441ba222b59ae796260373fd7f0cc6 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 4 May 2016 18:24:03 +0300
Subject: [PATCH 206/450] lib: istream's max_buffer_size=0 means 0, not
 unlimited.

Make sure we don't grow the buffer size then.
---
 src/lib/istream.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/lib/istream.c b/src/lib/istream.c
index c5bf2d6c29e..de2c95582c2 100644
--- a/src/lib/istream.c
+++ b/src/lib/istream.c
@@ -622,8 +622,7 @@ bool i_stream_try_alloc(struct istream_private *stream,
 		if (stream->skip > 0) {
 			/* remove the unused bytes from beginning of buffer */
                         i_stream_compress(stream);
-		} else if (stream->max_buffer_size == 0 ||
-			   stream->buffer_size < stream->max_buffer_size) {
+		} else if (stream->buffer_size < stream->max_buffer_size) {
 			/* buffer is full - grow it */
 			i_stream_grow_buffer(stream, I_STREAM_MIN_SIZE);
 		}

From a69f79313c926c2597c46d1e49356725c7955a38 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Sat, 14 May 2016 22:49:53 +0200
Subject: [PATCH 207/450] lib-mail: istream-qp-decoder: For efficiency, avoid
 copying data between decode buffer and stream buffer.

Instead, use the decode buffer directly as stream buffer.
---
 src/lib-mail/istream-qp-decoder.c | 38 +++++++++++++++++++++++--------
 1 file changed, 28 insertions(+), 10 deletions(-)

diff --git a/src/lib-mail/istream-qp-decoder.c b/src/lib-mail/istream-qp-decoder.c
index 3dfdd74dacd..f41c12fb2cd 100644
--- a/src/lib-mail/istream-qp-decoder.c
+++ b/src/lib-mail/istream-qp-decoder.c
@@ -32,24 +32,42 @@ static ssize_t i_stream_qp_decoder_read(struct istream_private *stream)
 	struct qp_decoder_istream *bstream =
 		(struct qp_decoder_istream *)stream;
 	const unsigned char *data;
-	size_t size, avail, error_pos;
+	size_t size, error_pos;
 	const char *error;
 	int ret;
 
 	for (;;) {
+		/* remove skipped data from buffer */
+		if (stream->skip > 0) {
+			i_assert(stream->skip <= bstream->buf->used);
+			buffer_delete(bstream->buf, 0, stream->skip);
+			stream->pos -= stream->skip;
+			stream->skip = 0;
+		}
+
+		stream->buffer = bstream->buf->data;
+
+		i_assert(stream->pos <= bstream->buf->used);
+		if (stream->pos >= bstream->istream.max_buffer_size) {
+			/* stream buffer still at maximum */
+			return -2;
+		}
+
 		/* if something is already decoded, return as much of it as
 		   we can */
 		if (bstream->buf->used > 0) {
-			i_stream_try_alloc(stream, bstream->buf->used, &avail);
-			if (avail == 0)
-				return -2;
-			size = I_MIN(avail, bstream->buf->used);
-			memcpy(stream->w_buffer + stream->pos,
-			       bstream->buf->data, size);
-			buffer_delete(bstream->buf, 0, size);
-			stream->pos += size;
-			return size;
+			size_t new_pos, bytes;
+
+			/* only return up to max_buffer_size bytes, even when buffer
+			   actually has more, as not to confuse the caller */
+			new_pos = I_MIN
+				(bstream->buf->used, bstream->istream.max_buffer_size);
+			bytes = new_pos - stream->pos;
+			stream->pos = new_pos;
+
+			return (ssize_t)bytes;
 		}
+
 		/* need to read more input */
 		ret = i_stream_read_data(stream->parent, &data, &size, 0);
 		if (ret <= 0) {

From 5f4111f3c49fff8c6bab322dd4b13d60ac79fe1b Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 1 Jun 2016 15:15:19 +0300
Subject: [PATCH 208/450] lib-fs: Added fs_write_stream_abort_async()

---
 src/lib-fs/fs-api.c | 7 +++++++
 src/lib-fs/fs-api.h | 3 +++
 2 files changed, 10 insertions(+)

diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c
index f57774676f6..fbf8fedcf78 100644
--- a/src/lib-fs/fs-api.c
+++ b/src/lib-fs/fs-api.c
@@ -692,6 +692,13 @@ void fs_write_stream_abort(struct fs_file *file, struct ostream **output)
 	(void)fs_write_stream_finish_int(file, FALSE);
 }
 
+void fs_write_stream_abort_async(struct fs_file *file)
+{
+	i_assert(file->output == NULL);
+
+	fs_write_stream_abort(file, &file->output);
+}
+
 void fs_write_set_hash(struct fs_file *file, const struct hash_method *method,
 		       const void *digest)
 {
diff --git a/src/lib-fs/fs-api.h b/src/lib-fs/fs-api.h
index 486e7fdf176..92a32fc5f42 100644
--- a/src/lib-fs/fs-api.h
+++ b/src/lib-fs/fs-api.h
@@ -270,6 +270,9 @@ int fs_write_stream_finish_async(struct fs_file *file);
    o_stream_ignore_last_errors() is called on the output stream so the caller
    doesn't need to do it. */
 void fs_write_stream_abort(struct fs_file *file, struct ostream **output);
+/* Abort writing to a stream after fs_write_stream_finish() was already
+   called. */
+void fs_write_stream_abort_async(struct fs_file *file);
 
 /* Set a hash to the following write. The storage can then verify that the
    input data matches the specified hash, or fail if it doesn't. Typically

From d0105748597c604474355f1a5441ddf105efce23 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 1 Jun 2016 17:06:28 +0300
Subject: [PATCH 209/450] lib-fs: Clarified fs_write_stream_abort() API and
 dropped its _async().

We can't handle an abort after an async fs_write_stream_finish() is already
going.
---
 src/lib-fs/fs-api-private.h |  2 ++
 src/lib-fs/fs-api.c         | 17 +++++++----------
 src/lib-fs/fs-api.h         |  7 +++----
 3 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/src/lib-fs/fs-api-private.h b/src/lib-fs/fs-api-private.h
index e37104e8a68..4b480adb361 100644
--- a/src/lib-fs/fs-api-private.h
+++ b/src/lib-fs/fs-api-private.h
@@ -48,6 +48,8 @@ struct fs_vfuncs {
 
 	int (*write)(struct fs_file *file, const void *data, size_t size);
 	void (*write_stream)(struct fs_file *file);
+	/* After write_stream_finish() is called once, all the following
+	   (async) calls will have success==TRUE. */
 	int (*write_stream_finish)(struct fs_file *file, bool success);
 
 	int (*lock)(struct fs_file *file, unsigned int secs,
diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c
index fbf8fedcf78..1f56603c3d5 100644
--- a/src/lib-fs/fs-api.c
+++ b/src/lib-fs/fs-api.c
@@ -682,21 +682,18 @@ int fs_write_stream_finish_async(struct fs_file *file)
 
 void fs_write_stream_abort(struct fs_file *file, struct ostream **output)
 {
+	int ret;
+
 	i_assert(*output == file->output);
+	i_assert(file->output != NULL);
+	i_assert(output != &file->output);
 
 	*output = NULL;
-	if (file->output != NULL)
-		o_stream_ignore_last_errors(file->output);
+	o_stream_ignore_last_errors(file->output);
 	/* make sure we don't have an old error lying around */
 	fs_set_error(file->fs, "Write aborted");
-	(void)fs_write_stream_finish_int(file, FALSE);
-}
-
-void fs_write_stream_abort_async(struct fs_file *file)
-{
-	i_assert(file->output == NULL);
-
-	fs_write_stream_abort(file, &file->output);
+	ret = fs_write_stream_finish_int(file, FALSE);
+	i_assert(ret != 0);
 }
 
 void fs_write_set_hash(struct fs_file *file, const struct hash_method *method,
diff --git a/src/lib-fs/fs-api.h b/src/lib-fs/fs-api.h
index 92a32fc5f42..d5702533f87 100644
--- a/src/lib-fs/fs-api.h
+++ b/src/lib-fs/fs-api.h
@@ -268,11 +268,10 @@ int fs_write_stream_finish(struct fs_file *file, struct ostream **output);
 int fs_write_stream_finish_async(struct fs_file *file);
 /* Abort writing via stream. Anything written to the stream is discarded.
    o_stream_ignore_last_errors() is called on the output stream so the caller
-   doesn't need to do it. */
+   doesn't need to do it. This must not be called after
+   fs_write_stream_finish(), i.e. it can't be used to abort a pending async
+   write. */
 void fs_write_stream_abort(struct fs_file *file, struct ostream **output);
-/* Abort writing to a stream after fs_write_stream_finish() was already
-   called. */
-void fs_write_stream_abort_async(struct fs_file *file);
 
 /* Set a hash to the following write. The storage can then verify that the
    input data matches the specified hash, or fail if it doesn't. Typically

From 90ec94c78789b0697d65cda8209879deb31323b6 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 1 Jun 2016 17:11:53 +0300
Subject: [PATCH 210/450] lib-fs: Added more asserts

---
 src/lib-fs/fs-api.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c
index 1f56603c3d5..024cec4ad62 100644
--- a/src/lib-fs/fs-api.c
+++ b/src/lib-fs/fs-api.c
@@ -651,8 +651,10 @@ static int fs_write_stream_finish_int(struct fs_file *file, bool success)
 		   indicated a failure. */
 		i_assert(success);
 	}
-	if (ret != 0)
+	if (ret != 0) {
+		i_assert(file->output == NULL);
 		file->writing_stream = FALSE;
+	}
 	return ret;
 }
 
@@ -661,6 +663,7 @@ int fs_write_stream_finish(struct fs_file *file, struct ostream **output)
 	bool success = TRUE;
 
 	i_assert(*output == file->output || *output == NULL);
+	i_assert(output != &file->output);
 
 	*output = NULL;
 	if (file->output != NULL) {

From f412ee7c6f0ef70cec05c9865c0320b7bbff761f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 1 Jun 2016 17:12:10 +0300
Subject: [PATCH 211/450] fs-metawrap: Removed unnecessary code.

The caller already set success==FALSE if ostream was closed.
---
 src/lib-fs/fs-metawrap.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/lib-fs/fs-metawrap.c b/src/lib-fs/fs-metawrap.c
index c2fd86677b1..0691fd0e2b8 100644
--- a/src/lib-fs/fs-metawrap.c
+++ b/src/lib-fs/fs-metawrap.c
@@ -383,8 +383,6 @@ static int fs_metawrap_write_stream_finish(struct fs_file *_file, bool success)
 	int ret;
 
 	if (_file->output != NULL) {
-		if (_file->output->closed)
-			success = FALSE;
 		if (_file->output == file->super_output)
 			_file->output = NULL;
 		else

From 1d29d49c74b47a53c098e01daf221bd6b79e35f9 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 1 Jun 2016 17:12:51 +0300
Subject: [PATCH 212/450] lib-fs: Improved unit tests.

---
 src/lib-fs/Makefile.am        |  1 +
 src/lib-fs/fs-test-async.c    | 97 +++++++++++++++++++++++++++++++++++
 src/lib-fs/fs-test.c          | 85 ++++++++++++++++++++++++++----
 src/lib-fs/fs-test.h          |  9 +++-
 src/lib-fs/test-fs-metawrap.c | 10 +++-
 5 files changed, 191 insertions(+), 11 deletions(-)
 create mode 100644 src/lib-fs/fs-test-async.c

diff --git a/src/lib-fs/Makefile.am b/src/lib-fs/Makefile.am
index 9a445cb949f..96930462eaa 100644
--- a/src/lib-fs/Makefile.am
+++ b/src/lib-fs/Makefile.am
@@ -14,6 +14,7 @@ libfs_la_SOURCES = \
 	fs-randomfail.c \
 	fs-posix.c \
 	fs-test.c \
+	fs-test-async.c \
 	fs-sis.c \
 	fs-sis-common.c \
 	fs-sis-queue.c \
diff --git a/src/lib-fs/fs-test-async.c b/src/lib-fs/fs-test-async.c
new file mode 100644
index 00000000000..2a73a74c768
--- /dev/null
+++ b/src/lib-fs/fs-test-async.c
@@ -0,0 +1,97 @@
+/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "ostream.h"
+#include "fs-test.h"
+#include "test-common.h"
+
+static void test_fs_async_write(const char *test_name, struct fs *fs)
+{
+	struct fs_file *file;
+	struct test_fs_file *test_file;
+	struct ostream *output;
+	unsigned int i;
+
+	test_begin(t_strdup_printf("%s: async write", test_name));
+	for (i = 0; i < 3; i++) {
+		file = fs_file_init(fs, "foo", FS_OPEN_MODE_REPLACE |
+				    FS_OPEN_FLAG_ASYNC);
+		output = fs_write_stream(file);
+
+		o_stream_nsend_str(output, "12345");
+		if (i < 2) {
+			test_assert(fs_write_stream_finish(file, &output) == 0);
+			test_assert(output == NULL);
+			test_assert(fs_write_stream_finish_async(file) == 0);
+		}
+
+		test_file = test_fs_file_get(fs, "foo");
+		test_file->wait_async = FALSE;
+
+		switch (i) {
+		case 0:
+			test_assert(fs_write_stream_finish_async(file) > 0);
+			test_assert(test_file->contents->used > 0);
+			break;
+		case 1:
+			test_file->io_failure = TRUE;
+			test_assert(fs_write_stream_finish_async(file) < 0);
+			test_assert(test_file->contents->used == 0);
+			break;
+		case 2:
+			fs_write_stream_abort(file, &output);
+			test_assert(test_file->contents->used == 0);
+			break;
+		}
+		fs_file_deinit(&file);
+	}
+	test_end();
+}
+
+static void test_fs_async_copy(const char *test_name, struct fs *fs)
+{
+	struct fs_file *src, *dest;
+	struct test_fs_file *test_file;
+
+	test_begin(t_strdup_printf("%s: async copy", test_name));
+
+	src = fs_file_init(fs, "foo", FS_OPEN_MODE_REPLACE);
+	test_assert(fs_write(src, "source", 6) == 0);
+
+	dest = fs_file_init(fs, "bar", FS_OPEN_MODE_REPLACE |
+			    FS_OPEN_FLAG_ASYNC);
+
+	test_assert(fs_copy(src, dest) == -1 && errno == EAGAIN);
+
+	test_file = test_fs_file_get(fs, "bar");
+	test_file->wait_async = FALSE;
+
+	test_assert(fs_copy_finish_async(dest) == 0);
+	test_assert(test_file->contents->used > 0);
+	fs_file_deinit(&dest);
+
+	fs_file_deinit(&src);
+	test_end();
+}
+
+void test_fs_async(const char *test_name, enum fs_properties properties,
+		   const char *driver, const char *args)
+{
+	struct fs_settings fs_set;
+	struct fs *fs;
+	struct test_fs *test_fs;
+	const char *error;
+
+	memset(&fs_set, 0, sizeof(fs_set));
+	if (fs_init(driver, args, &fs_set, &fs, &error) < 0)
+		i_fatal("fs_init() failed: %s", error);
+
+	test_fs = test_fs_get(fs);
+	test_fs->properties = properties;
+
+	test_fs_async_write(test_name, fs);
+	test_fs_async_copy(test_name, fs);
+
+	fs_deinit(&fs);
+}
diff --git a/src/lib-fs/fs-test.c b/src/lib-fs/fs-test.c
index 211b61f4129..e0627797df9 100644
--- a/src/lib-fs/fs-test.c
+++ b/src/lib-fs/fs-test.c
@@ -40,17 +40,19 @@ static enum fs_properties fs_test_get_properties(struct fs *_fs)
 
 static struct fs_file *
 fs_test_file_init(struct fs *_fs, const char *path,
-		  enum fs_open_mode mode, enum fs_open_flags flags ATTR_UNUSED)
+		  enum fs_open_mode mode, enum fs_open_flags flags)
 {
 	struct test_fs_file *file;
 
 	file = i_new(struct test_fs_file, 1);
 	file->file.fs = _fs;
 	file->file.path = i_strdup(path);
+	file->file.flags = flags;
 	file->mode = mode;
 	file->contents = buffer_create_dynamic(default_pool, 1024);
 	file->exists = TRUE;
 	file->seekable = TRUE;
+	file->wait_async = (flags & FS_OPEN_FLAG_ASYNC) != 0;
 	return &file->file;
 }
 
@@ -102,6 +104,16 @@ static int
 fs_test_get_metadata(struct fs_file *_file,
 		     const ARRAY_TYPE(fs_metadata) **metadata_r)
 {
+	struct test_fs_file *file = (struct test_fs_file *)_file;
+
+	if (file->wait_async) {
+		fs_set_error_async(_file->fs);
+		return -1;
+	}
+	if (file->io_failure) {
+		errno = EIO;
+		return -1;
+	}
 	fs_metadata_init(_file);
 	*metadata_r = &_file->metadata;
 	return 0;
@@ -132,6 +144,8 @@ fs_test_read_stream(struct fs_file *_file, size_t max_buffer_size ATTR_UNUSED)
 
 	if (!file->exists)
 		return i_stream_create_error(ENOENT);
+	if (file->io_failure)
+		return i_stream_create_error(EIO);
 	input = test_istream_create_data(file->contents->data,
 					 file->contents->used);
 	i_stream_add_destroy_callback(input, fs_test_stream_destroyed, file);
@@ -155,6 +169,14 @@ static int fs_test_write_stream_finish(struct fs_file *_file, bool success)
 {
 	struct test_fs_file *file = (struct test_fs_file *)_file;
 
+	if (_file->output != NULL)
+		o_stream_destroy(&_file->output);
+	if (file->wait_async) {
+		fs_set_error_async(_file->fs);
+		return 0;
+	}
+	if (file->io_failure)
+		success = FALSE;
 	if (!success)
 		buffer_set_used_size(file->contents, 0);
 	return success ? 1 : -1;
@@ -186,6 +208,14 @@ static int fs_test_exists(struct fs_file *_file)
 {
 	struct test_fs_file *file = (struct test_fs_file *)_file;
 
+	if (file->wait_async) {
+		fs_set_error_async(_file->fs);
+		return -1;
+	}
+	if (file->io_failure) {
+		errno = EIO;
+		return -1;
+	}
 	return file->exists ? 1 : 0;
 }
 
@@ -193,6 +223,14 @@ static int fs_test_stat(struct fs_file *_file, struct stat *st_r)
 {
 	struct test_fs_file *file = (struct test_fs_file *)_file;
 
+	if (file->wait_async) {
+		fs_set_error_async(_file->fs);
+		return -1;
+	}
+	if (file->io_failure) {
+		errno = EIO;
+		return -1;
+	}
 	if (!file->exists) {
 		errno = ENOENT;
 		return -1;
@@ -204,9 +242,22 @@ static int fs_test_stat(struct fs_file *_file, struct stat *st_r)
 
 static int fs_test_copy(struct fs_file *_src, struct fs_file *_dest)
 {
-	struct test_fs_file *src = (struct test_fs_file *)_src;
+	struct test_fs_file *src;
 	struct test_fs_file *dest = (struct test_fs_file *)_dest;
 
+	if (_src != NULL)
+		dest->copy_src = test_fs_file_get(_src->fs, fs_file_path(_src));
+	src = dest->copy_src;
+	if (dest->wait_async) {
+		fs_set_error_async(_dest->fs);
+		return -1;
+	}
+	dest->copy_src = NULL;
+
+	if (dest->io_failure) {
+		errno = EIO;
+		return -1;
+	}
 	if (!src->exists) {
 		errno = ENOENT;
 		return -1;
@@ -220,6 +271,12 @@ static int fs_test_copy(struct fs_file *_src, struct fs_file *_dest)
 static int fs_test_rename(struct fs_file *_src, struct fs_file *_dest)
 {
 	struct test_fs_file *src = (struct test_fs_file *)_src;
+	struct test_fs_file *dest = (struct test_fs_file *)_dest;
+
+	if (src->wait_async || dest->wait_async) {
+		fs_set_error_async(_dest->fs);
+		return -1;
+	}
 
 	if (fs_test_copy(_src, _dest) < 0)
 		return -1;
@@ -231,6 +288,11 @@ static int fs_test_delete(struct fs_file *_file)
 {
 	struct test_fs_file *file = (struct test_fs_file *)_file;
 
+	if (file->wait_async) {
+		fs_set_error_async(_file->fs);
+		return -1;
+	}
+
 	if (!file->exists) {
 		errno = ENOENT;
 		return -1;
@@ -302,21 +364,26 @@ static int fs_test_iter_deinit(struct fs_iter *_iter)
 	return ret;
 }
 
-struct test_fs_file *test_fs_file_get(struct fs *fs, unsigned int n)
+struct test_fs *test_fs_get(struct fs *fs)
 {
-	struct fs_file *file;
-
 	while (strcmp(fs->name, "test") != 0) {
 		i_assert(fs->parent != NULL);
 		fs = fs->parent;
 	}
+	return (struct test_fs *)fs;
+}
+
+struct test_fs_file *test_fs_file_get(struct fs *fs, const char *path)
+{
+	struct fs_file *file;
+
+	fs = &test_fs_get(fs)->fs;
 
-	file = fs->files;
-	for (; n > 0; n--) {
+	for (file = fs->files;; file = file->next) {
 		i_assert(file != NULL);
-		file = file->next;
+		if (strcmp(fs_file_path(file), path) == 0)
+			break;
 	}
-	i_assert(file != NULL);
 	return (struct test_fs_file *)file;
 }
 
diff --git a/src/lib-fs/fs-test.h b/src/lib-fs/fs-test.h
index c5f94cc4f2c..5a2ed1434a6 100644
--- a/src/lib-fs/fs-test.h
+++ b/src/lib-fs/fs-test.h
@@ -18,12 +18,15 @@ struct test_fs_file {
 
 	buffer_t *contents;
 	struct istream *input;
+	struct test_fs_file *copy_src;
 
 	bool prefetched;
 	bool locked;
 	bool exists;
 	bool seekable;
 	bool closed;
+	bool io_failure;
+	bool wait_async;
 };
 
 struct test_fs_iter {
@@ -33,6 +36,10 @@ struct test_fs_iter {
 	bool failed;
 };
 
-struct test_fs_file *test_fs_file_get(struct fs *fs, unsigned int n);
+struct test_fs *test_fs_get(struct fs *fs);
+struct test_fs_file *test_fs_file_get(struct fs *fs, const char *path);
+
+void test_fs_async(const char *test_name, enum fs_properties properties,
+		   const char *driver, const char *args);
 
 #endif
diff --git a/src/lib-fs/test-fs-metawrap.c b/src/lib-fs/test-fs-metawrap.c
index b09c92dad11..6b1f52db7a7 100644
--- a/src/lib-fs/test-fs-metawrap.c
+++ b/src/lib-fs/test-fs-metawrap.c
@@ -26,7 +26,7 @@ static void test_fs_metawrap_stat(void)
 	for (i = 0; i < 2; i++) {
 		file = fs_file_init(fs, "foo", FS_OPEN_MODE_READONLY);
 
-		test_file = test_fs_file_get(fs, 0);
+		test_file = test_fs_file_get(fs, "foo");
 		str_append(test_file->contents, "key:value\n\n12345678901234567890");
 
 		if (i == 0) {
@@ -46,10 +46,18 @@ static void test_fs_metawrap_stat(void)
 	test_end();
 }
 
+static void test_fs_metawrap_async(void)
+{
+	test_fs_async("metawrap", FS_PROPERTY_METADATA, "metawrap", "test");
+	test_fs_async("metawrap passthrough", 0, "metawrap", "test");
+	test_fs_async("double-metawrap", FS_PROPERTY_METADATA, "metawrap", "metawrap:test");
+}
+
 int main(void)
 {
 	static void (*test_functions[])(void) = {
 		test_fs_metawrap_stat,
+		test_fs_metawrap_async,
 		NULL
 	};
 	return test_run(test_functions);

From 3ffc655f0a4f8db2315c3de3553156f8bf99db75 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 1 Jun 2016 17:18:23 +0300
Subject: [PATCH 213/450] fs-metawrap: Removed unnecessary code.

As mentioned in a0cf7d392, this can't happen.
---
 src/lib-fs/fs-metawrap.c | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/src/lib-fs/fs-metawrap.c b/src/lib-fs/fs-metawrap.c
index 0691fd0e2b8..c4887d75c25 100644
--- a/src/lib-fs/fs-metawrap.c
+++ b/src/lib-fs/fs-metawrap.c
@@ -393,11 +393,8 @@ static int fs_metawrap_write_stream_finish(struct fs_file *_file, bool success)
 			/* no metawrap */
 			i_assert(file->temp_output == NULL);
 			fs_write_stream_abort(file->super, &file->super_output);
-		} else if (file->temp_output == NULL) {
-			/* finishing up */
-			i_assert(file->super_output == NULL);
-			fs_write_stream_abort(file->super, &file->super_output);
 		} else {
+			i_assert(file->temp_output != NULL);
 			o_stream_destroy(&file->temp_output);
 		}
 		return -1;

From 6bd205f9a45cf649d2174292c4feb35bba5b7943 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 1 Jun 2016 17:52:23 +0300
Subject: [PATCH 214/450] lib-compression: istream.max_buffer_size == 0 isn't
 unlimited.

---
 src/lib-compression/istream-bzlib.c | 3 +--
 src/lib-compression/istream-lzma.c  | 3 +--
 src/lib-compression/istream-zlib.c  | 3 +--
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/lib-compression/istream-bzlib.c b/src/lib-compression/istream-bzlib.c
index b8d0dff86cc..42ba6da72b4 100644
--- a/src/lib-compression/istream-bzlib.c
+++ b/src/lib-compression/istream-bzlib.c
@@ -86,8 +86,7 @@ static ssize_t i_stream_bzlib_read(struct istream_private *stream)
 			   have a seek mark. */
 			i_stream_compress(stream);
 		}
-		if (stream->max_buffer_size == 0 ||
-		    stream->buffer_size < stream->max_buffer_size)
+		if (stream->buffer_size < stream->max_buffer_size)
 			i_stream_grow_buffer(stream, CHUNK_SIZE);
 
 		if (stream->pos == stream->buffer_size) {
diff --git a/src/lib-compression/istream-lzma.c b/src/lib-compression/istream-lzma.c
index 865a7819d55..282f0d0df58 100644
--- a/src/lib-compression/istream-lzma.c
+++ b/src/lib-compression/istream-lzma.c
@@ -95,8 +95,7 @@ static ssize_t i_stream_lzma_read(struct istream_private *stream)
 			   have a seek mark. */
 			i_stream_compress(stream);
 		}
-		if (stream->max_buffer_size == 0 ||
-		    stream->buffer_size < stream->max_buffer_size)
+		if (stream->buffer_size < stream->max_buffer_size)
 			i_stream_grow_buffer(stream, CHUNK_SIZE);
 
 		if (stream->pos == stream->buffer_size) {
diff --git a/src/lib-compression/istream-zlib.c b/src/lib-compression/istream-zlib.c
index d212a20ce2c..94d9cf78f9d 100644
--- a/src/lib-compression/istream-zlib.c
+++ b/src/lib-compression/istream-zlib.c
@@ -230,8 +230,7 @@ static ssize_t i_stream_zlib_read(struct istream_private *stream)
 			   have a seek mark. */
 			i_stream_compress(stream);
 		}
-		if (stream->max_buffer_size == 0 ||
-		    stream->buffer_size < stream->max_buffer_size)
+		if (stream->buffer_size < stream->max_buffer_size)
 			i_stream_grow_buffer(stream, CHUNK_SIZE);
 
 		if (stream->pos == stream->buffer_size) {

From 51dcb33a339e369de7c6e62b77f049c77bdb1f7e Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 1 Jun 2016 18:03:58 +0300
Subject: [PATCH 215/450] lib: istream-chain didn't update its max_buffer_size

I had this code at some point earlier, but looks like it got lost from the
final commit.
---
 src/lib/istream-chain.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/lib/istream-chain.c b/src/lib/istream-chain.c
index ddf0f566c82..6880f8d00e2 100644
--- a/src/lib/istream-chain.c
+++ b/src/lib/istream-chain.c
@@ -51,6 +51,11 @@ i_stream_chain_append_internal(struct istream_chain *chain,
 		if (cstream->have_explicit_max_buffer_size) {
 			i_stream_set_max_buffer_size(stream,
 				chain->stream->istream.max_buffer_size);
+		} else {
+			size_t max_size = i_stream_get_max_buffer_size(stream);
+
+			if (cstream->istream.max_buffer_size < max_size)
+				cstream->istream.max_buffer_size = max_size;
 		}
 	}
 	DLLIST2_APPEND(&chain->head, &chain->tail, link);

From 41a18c16c728751acb5efc002bff64b433c18de5 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 1 Jun 2016 18:09:48 +0300
Subject: [PATCH 216/450] lib: i_stream_get_max_buffer_size() checks also
 parents' max sizes

This fixes i_stream_get_max_buffer_size() to work correctly with
istream-chain.
---
 src/lib/istream.c | 9 ++++++++-
 src/lib/istream.h | 8 ++++++--
 2 files changed, 14 insertions(+), 3 deletions(-)

diff --git a/src/lib/istream.c b/src/lib/istream.c
index de2c95582c2..d235218b909 100644
--- a/src/lib/istream.c
+++ b/src/lib/istream.c
@@ -119,7 +119,14 @@ void i_stream_set_max_buffer_size(struct istream *stream, size_t max_size)
 
 size_t i_stream_get_max_buffer_size(struct istream *stream)
 {
-	return stream->real_stream->max_buffer_size;
+	size_t max_size = 0;
+
+	do {
+		if (max_size < stream->real_stream->max_buffer_size)
+			max_size = stream->real_stream->max_buffer_size;
+		stream = stream->real_stream->parent;
+	} while (stream != NULL);
+	return max_size;
 }
 
 void i_stream_set_return_partial_line(struct istream *stream, bool set)
diff --git a/src/lib/istream.h b/src/lib/istream.h
index 4db812005d9..34f95715d1d 100644
--- a/src/lib/istream.h
+++ b/src/lib/istream.h
@@ -94,9 +94,13 @@ void i_stream_sync(struct istream *stream);
    unless it's called before reading anything. */
 void i_stream_set_init_buffer_size(struct istream *stream, size_t size);
 /* Change the maximum size for stream's input buffer to grow. Useful only
-   for buffered streams (currently only file). */
+   for buffered streams (currently only file). This changes also all the
+   parent streams' max buffer size. */
 void i_stream_set_max_buffer_size(struct istream *stream, size_t max_size);
-/* Returns the current max. buffer size. */
+/* Returns the current max. buffer size for the stream. This function also
+   goesthrough all of the parent streams and returns the highest seen max
+   buffer size. This is needed because some streams (e.g. istream-chain) change
+   their max buffer size dynamically. */
 size_t i_stream_get_max_buffer_size(struct istream *stream);
 /* Enable/disable i_stream[_read]_next_line() returning the last line if it
    doesn't end with LF. */

From f67f6de07a5e3120e25b3eaaed5730cdb7a25f1d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 1 Jun 2016 18:11:29 +0300
Subject: [PATCH 217/450] global: Use i_stream_get_max_buffer_size() wherever
 possible

After 7be8ba0c0 it's no longer correct to access stream->max_buffer_size
directly. These changes fix using istream-chain with various wrapper
streams.
---
 src/lib-compression/istream-bzlib.c         |  2 +-
 src/lib-compression/istream-lz4.c           |  2 +-
 src/lib-compression/istream-lzma.c          |  2 +-
 src/lib-compression/istream-zlib.c          |  2 +-
 src/lib-fs/istream-fs-file.c                |  2 +-
 src/lib-mail/istream-attachment-extractor.c |  2 +-
 src/lib-mail/istream-binary-converter.c     |  2 +-
 src/lib-mail/istream-header-filter.c        |  6 ++++--
 src/lib-mail/istream-qp-decoder.c           |  8 ++++----
 src/lib-ssl-iostream/istream-openssl.c      | 11 ++++++-----
 src/lib-test/test-common.c                  |  4 +++-
 src/lib/istream-concat.c                    |  4 ++--
 src/lib/istream.c                           | 12 ++++++------
 13 files changed, 32 insertions(+), 27 deletions(-)

diff --git a/src/lib-compression/istream-bzlib.c b/src/lib-compression/istream-bzlib.c
index 42ba6da72b4..cb3db16ac8b 100644
--- a/src/lib-compression/istream-bzlib.c
+++ b/src/lib-compression/istream-bzlib.c
@@ -86,7 +86,7 @@ static ssize_t i_stream_bzlib_read(struct istream_private *stream)
 			   have a seek mark. */
 			i_stream_compress(stream);
 		}
-		if (stream->buffer_size < stream->max_buffer_size)
+		if (stream->buffer_size < i_stream_get_max_buffer_size(&stream->istream))
 			i_stream_grow_buffer(stream, CHUNK_SIZE);
 
 		if (stream->pos == stream->buffer_size) {
diff --git a/src/lib-compression/istream-lz4.c b/src/lib-compression/istream-lz4.c
index d53c970caa6..ded44dc8ce7 100644
--- a/src/lib-compression/istream-lz4.c
+++ b/src/lib-compression/istream-lz4.c
@@ -150,7 +150,7 @@ static ssize_t i_stream_lz4_read(struct istream_private *stream)
 	}
 	/* if we already have max_buffer_size amount of data, fail here */
 	i_stream_compress(stream);
-	if (stream->pos >= stream->max_buffer_size)
+	if (stream->pos >= i_stream_get_max_buffer_size(&stream->istream))
 		return -2;
 	/* allocate enough space for the old data and the new
 	   decompressed chunk. we don't know the original compressed size,
diff --git a/src/lib-compression/istream-lzma.c b/src/lib-compression/istream-lzma.c
index 282f0d0df58..20bb93b17ea 100644
--- a/src/lib-compression/istream-lzma.c
+++ b/src/lib-compression/istream-lzma.c
@@ -95,7 +95,7 @@ static ssize_t i_stream_lzma_read(struct istream_private *stream)
 			   have a seek mark. */
 			i_stream_compress(stream);
 		}
-		if (stream->buffer_size < stream->max_buffer_size)
+		if (stream->buffer_size < i_stream_get_max_buffer_size(&stream->istream))
 			i_stream_grow_buffer(stream, CHUNK_SIZE);
 
 		if (stream->pos == stream->buffer_size) {
diff --git a/src/lib-compression/istream-zlib.c b/src/lib-compression/istream-zlib.c
index 94d9cf78f9d..b32a1d73e9b 100644
--- a/src/lib-compression/istream-zlib.c
+++ b/src/lib-compression/istream-zlib.c
@@ -230,7 +230,7 @@ static ssize_t i_stream_zlib_read(struct istream_private *stream)
 			   have a seek mark. */
 			i_stream_compress(stream);
 		}
-		if (stream->buffer_size < stream->max_buffer_size)
+		if (stream->buffer_size < i_stream_get_max_buffer_size(&stream->istream))
 			i_stream_grow_buffer(stream, CHUNK_SIZE);
 
 		if (stream->pos == stream->buffer_size) {
diff --git a/src/lib-fs/istream-fs-file.c b/src/lib-fs/istream-fs-file.c
index 4dc5c2d153c..e7aecb3c13e 100644
--- a/src/lib-fs/istream-fs-file.c
+++ b/src/lib-fs/istream-fs-file.c
@@ -27,7 +27,7 @@ static ssize_t i_stream_fs_file_read(struct istream_private *stream)
 
 	if (fstream->istream.parent == NULL) {
 		input = fs_read_stream(fstream->file,
-				       fstream->istream.max_buffer_size);
+			i_stream_get_max_buffer_size(&stream->istream));
 		i_stream_init_parent(stream, input);
 		i_stream_unref(&input);
 	}
diff --git a/src/lib-mail/istream-attachment-extractor.c b/src/lib-mail/istream-attachment-extractor.c
index bc62146d24f..09a21ee7cc8 100644
--- a/src/lib-mail/istream-attachment-extractor.c
+++ b/src/lib-mail/istream-attachment-extractor.c
@@ -594,7 +594,7 @@ static int astream_read_next(struct attachment_istream *astream, bool *retry_r)
 
 	*retry_r = FALSE;
 
-	if (stream->pos - stream->skip >= stream->max_buffer_size)
+	if (stream->pos - stream->skip >= i_stream_get_max_buffer_size(&stream->istream))
 		return -2;
 
 	if (astream->failed) {
diff --git a/src/lib-mail/istream-binary-converter.c b/src/lib-mail/istream-binary-converter.c
index 87f2f419a13..f2564422f37 100644
--- a/src/lib-mail/istream-binary-converter.c
+++ b/src/lib-mail/istream-binary-converter.c
@@ -180,7 +180,7 @@ static ssize_t i_stream_binary_converter_read(struct istream_private *stream)
 	struct message_block block;
 	size_t old_size, new_size;
 
-	if (stream->pos - stream->skip >= stream->max_buffer_size)
+	if (stream->pos - stream->skip >= i_stream_get_max_buffer_size(&stream->istream))
 		return -2;
 	old_size = stream->pos - stream->skip;
 
diff --git a/src/lib-mail/istream-header-filter.c b/src/lib-mail/istream-header-filter.c
index 19b671a247c..f83c02c15d2 100644
--- a/src/lib-mail/istream-header-filter.c
+++ b/src/lib-mail/istream-header-filter.c
@@ -157,6 +157,7 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 {
 	struct message_header_line *hdr;
 	uoff_t highwater_offset;
+	size_t max_buffer_size;
 	ssize_t ret, ret2;
 	int hdr_ret;
 
@@ -186,7 +187,8 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 		}
 	}
 
-	if (mstream->hdr_buf->used >= mstream->istream.max_buffer_size)
+	max_buffer_size = i_stream_get_max_buffer_size(&mstream->istream.istream);
+	if (mstream->hdr_buf->used >= max_buffer_size)
 		return -2;
 
 	while ((hdr_ret = message_parse_header_next(mstream->hdr_ctx,
@@ -286,7 +288,7 @@ static ssize_t read_header(struct header_filter_istream *mstream)
 				break;
 			}
 		}
-		if (mstream->hdr_buf->used >= mstream->istream.max_buffer_size)
+		if (mstream->hdr_buf->used >= max_buffer_size)
 			break;
 	}
 	if (mstream->hdr_buf->used > 0) {
diff --git a/src/lib-mail/istream-qp-decoder.c b/src/lib-mail/istream-qp-decoder.c
index f41c12fb2cd..611defe5cb0 100644
--- a/src/lib-mail/istream-qp-decoder.c
+++ b/src/lib-mail/istream-qp-decoder.c
@@ -32,10 +32,11 @@ static ssize_t i_stream_qp_decoder_read(struct istream_private *stream)
 	struct qp_decoder_istream *bstream =
 		(struct qp_decoder_istream *)stream;
 	const unsigned char *data;
-	size_t size, error_pos;
+	size_t size, error_pos, max_buffer_size;
 	const char *error;
 	int ret;
 
+	max_buffer_size = i_stream_get_max_buffer_size(&stream->istream);
 	for (;;) {
 		/* remove skipped data from buffer */
 		if (stream->skip > 0) {
@@ -48,7 +49,7 @@ static ssize_t i_stream_qp_decoder_read(struct istream_private *stream)
 		stream->buffer = bstream->buf->data;
 
 		i_assert(stream->pos <= bstream->buf->used);
-		if (stream->pos >= bstream->istream.max_buffer_size) {
+		if (stream->pos >= max_buffer_size) {
 			/* stream buffer still at maximum */
 			return -2;
 		}
@@ -60,8 +61,7 @@ static ssize_t i_stream_qp_decoder_read(struct istream_private *stream)
 
 			/* only return up to max_buffer_size bytes, even when buffer
 			   actually has more, as not to confuse the caller */
-			new_pos = I_MIN
-				(bstream->buf->used, bstream->istream.max_buffer_size);
+			new_pos = I_MIN(bstream->buf->used, max_buffer_size);
 			bytes = new_pos - stream->pos;
 			stream->pos = new_pos;
 
diff --git a/src/lib-ssl-iostream/istream-openssl.c b/src/lib-ssl-iostream/istream-openssl.c
index bdfc21c9dd2..51b6c265990 100644
--- a/src/lib-ssl-iostream/istream-openssl.c
+++ b/src/lib-ssl-iostream/istream-openssl.c
@@ -33,6 +33,7 @@ static ssize_t i_stream_ssl_read_real(struct istream_private *stream)
 	struct ssl_istream *sstream = (struct ssl_istream *)stream;
 	struct ssl_iostream *ssl_io = sstream->ssl_io;
 	unsigned char buffer[IO_BLOCK_SIZE];
+	size_t max_buffer_size = i_stream_get_max_buffer_size(&stream->istream);
 	size_t orig_max_buffer_size = stream->max_buffer_size;
 	size_t size;
 	ssize_t ret, total_ret;
@@ -42,9 +43,9 @@ static ssize_t i_stream_ssl_read_real(struct istream_private *stream)
 		return -1;
 	}
 
-	if (stream->pos >= stream->max_buffer_size) {
+	if (stream->pos >= max_buffer_size) {
 		i_stream_compress(stream);
-		if (stream->pos >= stream->max_buffer_size)
+		if (stream->pos >= max_buffer_size)
 			return -2;
 	}
 
@@ -62,9 +63,9 @@ static ssize_t i_stream_ssl_read_real(struct istream_private *stream)
 
 	if (!i_stream_try_alloc(stream, 1, &size))
 		i_unreached();
-	if (stream->pos + size > stream->max_buffer_size) {
-		i_assert(stream->max_buffer_size > stream->pos);
-		size = stream->max_buffer_size - stream->pos;
+	if (stream->pos + size > max_buffer_size) {
+		i_assert(max_buffer_size > stream->pos);
+		size = max_buffer_size - stream->pos;
 	}
 
 	while ((ret = SSL_read(ssl_io->ssl,
diff --git a/src/lib-test/test-common.c b/src/lib-test/test-common.c
index aff3c230da8..04e61197acc 100644
--- a/src/lib-test/test-common.c
+++ b/src/lib-test/test-common.c
@@ -38,8 +38,10 @@ static ssize_t test_read(struct istream_private *stream)
 
 	i_assert(stream->skip <= stream->pos);
 
-	if (stream->pos - stream->skip >= tstream->istream.max_buffer_size)
+	if (stream->pos - stream->skip >= tstream->istream.max_buffer_size) {
+		i_assert(stream->skip != stream->pos);
 		return -2;
+	}
 
 	if (tstream->max_pos < stream->pos) {
 		/* we seeked past the end of file. */
diff --git a/src/lib/istream-concat.c b/src/lib/istream-concat.c
index fb9c20188ec..a6937045746 100644
--- a/src/lib/istream-concat.c
+++ b/src/lib/istream-concat.c
@@ -169,7 +169,7 @@ static ssize_t i_stream_concat_read(struct istream_private *stream)
 		/* we either read something or we're at EOF */
 		last_stream = cstream->input[cstream->cur_idx+1] == NULL;
 		if (ret == -1 && !last_stream) {
-			if (stream->pos - stream->skip >= stream->max_buffer_size)
+			if (stream->pos - stream->skip >= i_stream_get_max_buffer_size(&stream->istream))
 				return -2;
 
 			i_stream_concat_read_next(cstream);
@@ -320,7 +320,7 @@ struct istream *i_stream_create_concat(struct istream *input[])
 	/* if any of the streams isn't blocking or seekable, set ourself also
 	   nonblocking/nonseekable */
 	for (count = 0; input[count] != NULL; count++) {
-		size_t cur_max = input[count]->real_stream->max_buffer_size;
+		size_t cur_max = i_stream_get_max_buffer_size(input[count]);
 
 		if (cur_max > max_buffer_size)
 			max_buffer_size = cur_max;
diff --git a/src/lib/istream.c b/src/lib/istream.c
index d235218b909..aa95c1cb9d7 100644
--- a/src/lib/istream.c
+++ b/src/lib/istream.c
@@ -596,9 +596,7 @@ void i_stream_compress(struct istream_private *stream)
 
 void i_stream_grow_buffer(struct istream_private *stream, size_t bytes)
 {
-	size_t old_size;
-
-	i_assert(stream->max_buffer_size > 0);
+	size_t old_size, max_size;
 
 	old_size = stream->buffer_size;
 
@@ -608,8 +606,10 @@ void i_stream_grow_buffer(struct istream_private *stream, size_t bytes)
 	else
 		stream->buffer_size = nearest_power(stream->buffer_size);
 
-	if (stream->buffer_size > stream->max_buffer_size)
-		stream->buffer_size = stream->max_buffer_size;
+	max_size = i_stream_get_max_buffer_size(&stream->istream);
+	i_assert(max_size > 0);
+	if (stream->buffer_size > max_size)
+		stream->buffer_size = max_size;
 
 	if (stream->buffer_size <= old_size)
 		stream->buffer_size = old_size;
@@ -629,7 +629,7 @@ bool i_stream_try_alloc(struct istream_private *stream,
 		if (stream->skip > 0) {
 			/* remove the unused bytes from beginning of buffer */
                         i_stream_compress(stream);
-		} else if (stream->buffer_size < stream->max_buffer_size) {
+		} else if (stream->buffer_size < i_stream_get_max_buffer_size(&stream->istream)) {
 			/* buffer is full - grow it */
 			i_stream_grow_buffer(stream, I_STREAM_MIN_SIZE);
 		}

From 72f0027d99c4441c6ceb89f13b1944e64b0f1f01 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 2 Jun 2016 00:51:26 +0300
Subject: [PATCH 218/450] dict: Fixed crash when reaching max (5) pending
 commands.

---
 src/dict/dict-connection.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/dict/dict-connection.c b/src/dict/dict-connection.c
index 35d8e9386b9..3040ec3132e 100644
--- a/src/dict/dict-connection.c
+++ b/src/dict/dict-connection.c
@@ -155,6 +155,7 @@ static void dict_connection_input(struct dict_connection *conn)
 			io_remove(&conn->io);
 			if (conn->to_input != NULL)
 				timeout_remove(&conn->to_input);
+			break;
 		}
 	}
 }

From 8b895d07ddd7044050267a5db3b1272183152307 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 2 Jun 2016 00:52:37 +0300
Subject: [PATCH 219/450] dict: Pipelined iteration replies may have been
 hanging.

For example:
 - lookup start
 - iterate start
 - iterate finished, but can't reply yet
 - lookup finished
 - iterate reply can be sent now, but wasn't previously
---
 src/dict/dict-commands.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/dict/dict-commands.c b/src/dict/dict-commands.c
index 45078144ca5..7c19f468eae 100644
--- a/src/dict/dict-commands.c
+++ b/src/dict/dict-commands.c
@@ -34,6 +34,8 @@ struct dict_connection_cmd {
 
 struct dict_command_stats cmd_stats;
 
+static int cmd_iterate_flush(struct dict_connection_cmd *cmd);
+
 static void dict_connection_cmd_output_more(struct dict_connection_cmd *cmd);
 
 static void dict_connection_cmd_free(struct dict_connection_cmd *cmd)
@@ -72,6 +74,10 @@ static void dict_connection_cmds_flush(struct dict_connection *conn)
 		first_cmdp = array_idx(&conn->cmds, 0);
 		cmd = *first_cmdp;
 
+		/* we may be able to start outputting iterations now. */
+		if (cmd->iter != NULL)
+			(void)cmd_iterate_flush(cmd);
+
 		if (cmd->reply == NULL) {
 			/* command not finished yet */
 			break;

From c7021f4200bce3cca050a35b8d0aa51f46456dc5 Mon Sep 17 00:00:00 2001
From: Teemu Huovila <teemu.huovila@dovecot.fi>
Date: Mon, 30 May 2016 11:54:26 +0300
Subject: [PATCH 220/450] lib-fts: Move stopwords to subdirectory.

All files incluided in dist are explicitly mentioned. The whole
subdirectory 'stopwords' could also be distributed, but that is
more error prone.
---
 src/lib-fts/Makefile.am                      | 12 ++++++------
 src/lib-fts/{ => stopwords}/stopwords_en.txt |  0
 src/lib-fts/{ => stopwords}/stopwords_fi.txt |  0
 src/lib-fts/{ => stopwords}/stopwords_fr.txt |  0
 src/lib-fts/{ => stopwords}/stopwords_no.txt |  0
 src/lib-fts/{ => stopwords}/stopwords_sv.txt |  0
 6 files changed, 6 insertions(+), 6 deletions(-)
 rename src/lib-fts/{ => stopwords}/stopwords_en.txt (100%)
 rename src/lib-fts/{ => stopwords}/stopwords_fi.txt (100%)
 rename src/lib-fts/{ => stopwords}/stopwords_fr.txt (100%)
 rename src/lib-fts/{ => stopwords}/stopwords_no.txt (100%)
 rename src/lib-fts/{ => stopwords}/stopwords_sv.txt (100%)

diff --git a/src/lib-fts/Makefile.am b/src/lib-fts/Makefile.am
index fa04b2bad05..fb19956eb75 100644
--- a/src/lib-fts/Makefile.am
+++ b/src/lib-fts/Makefile.am
@@ -12,15 +12,15 @@ AM_CPPFLAGS = \
 	$(LIBICU_CFLAGS) \
 	-DUDHRDIR=\""$(top_srcdir)/src/lib-fts"\" \
 	-DDATADIR=\"$(pkgdatadir)\" \
-	-DTEST_STOPWORDS_DIR=\""$(top_srcdir)/src/lib-fts"\"
+	-DTEST_STOPWORDS_DIR=\""$(top_srcdir)/src/lib-fts/stopwords"\"
 
 stopwordsdir = $(datadir)/${PACKAGE_TARNAME}/stopwords
 dist_stopwords_DATA = \
-	stopwords_en.txt \
-	stopwords_fi.txt \
-	stopwords_fr.txt \
-	stopwords_no.txt \
-	stopwords_sv.txt
+	stopwords/stopwords_en.txt \
+	stopwords/stopwords_fi.txt \
+	stopwords/stopwords_fr.txt \
+	stopwords/stopwords_no.txt \
+	stopwords/stopwords_sv.txt
 
 BUILT_SOURCES = word-boundary-data.c word-break-data.c
 
diff --git a/src/lib-fts/stopwords_en.txt b/src/lib-fts/stopwords/stopwords_en.txt
similarity index 100%
rename from src/lib-fts/stopwords_en.txt
rename to src/lib-fts/stopwords/stopwords_en.txt
diff --git a/src/lib-fts/stopwords_fi.txt b/src/lib-fts/stopwords/stopwords_fi.txt
similarity index 100%
rename from src/lib-fts/stopwords_fi.txt
rename to src/lib-fts/stopwords/stopwords_fi.txt
diff --git a/src/lib-fts/stopwords_fr.txt b/src/lib-fts/stopwords/stopwords_fr.txt
similarity index 100%
rename from src/lib-fts/stopwords_fr.txt
rename to src/lib-fts/stopwords/stopwords_fr.txt
diff --git a/src/lib-fts/stopwords_no.txt b/src/lib-fts/stopwords/stopwords_no.txt
similarity index 100%
rename from src/lib-fts/stopwords_no.txt
rename to src/lib-fts/stopwords/stopwords_no.txt
diff --git a/src/lib-fts/stopwords_sv.txt b/src/lib-fts/stopwords/stopwords_sv.txt
similarity index 100%
rename from src/lib-fts/stopwords_sv.txt
rename to src/lib-fts/stopwords/stopwords_sv.txt

From 77c9af3438e7cc07379063a475e9e16c38a556f3 Mon Sep 17 00:00:00 2001
From: Teemu Huovila <teemu.huovila@dovecot.fi>
Date: Mon, 30 May 2016 12:40:00 +0300
Subject: [PATCH 221/450] lib-fts: Add stopword files for more languages.

---
 src/lib-fts/Makefile.am                |   8 +
 src/lib-fts/stopwords/stopwords_da.txt | 109 ++++++++
 src/lib-fts/stopwords/stopwords_de.txt | 293 ++++++++++++++++++++
 src/lib-fts/stopwords/stopwords_es.txt | 355 +++++++++++++++++++++++++
 src/lib-fts/stopwords/stopwords_it.txt | 302 +++++++++++++++++++++
 src/lib-fts/stopwords/stopwords_nl.txt | 118 ++++++++
 src/lib-fts/stopwords/stopwords_pt.txt | 252 ++++++++++++++++++
 src/lib-fts/stopwords/stopwords_ro.txt | 233 ++++++++++++++++
 src/lib-fts/stopwords/stopwords_ru.txt | 242 +++++++++++++++++
 9 files changed, 1912 insertions(+)
 create mode 100644 src/lib-fts/stopwords/stopwords_da.txt
 create mode 100644 src/lib-fts/stopwords/stopwords_de.txt
 create mode 100644 src/lib-fts/stopwords/stopwords_es.txt
 create mode 100644 src/lib-fts/stopwords/stopwords_it.txt
 create mode 100644 src/lib-fts/stopwords/stopwords_nl.txt
 create mode 100644 src/lib-fts/stopwords/stopwords_pt.txt
 create mode 100644 src/lib-fts/stopwords/stopwords_ro.txt
 create mode 100644 src/lib-fts/stopwords/stopwords_ru.txt

diff --git a/src/lib-fts/Makefile.am b/src/lib-fts/Makefile.am
index fb19956eb75..4eb3f49e939 100644
--- a/src/lib-fts/Makefile.am
+++ b/src/lib-fts/Makefile.am
@@ -16,10 +16,18 @@ AM_CPPFLAGS = \
 
 stopwordsdir = $(datadir)/${PACKAGE_TARNAME}/stopwords
 dist_stopwords_DATA = \
+	stopwords/stopwords_da.txt \
+	stopwords/stopwords_de.txt \
 	stopwords/stopwords_en.txt \
+	stopwords/stopwords_es.txt \
 	stopwords/stopwords_fi.txt \
 	stopwords/stopwords_fr.txt \
+	stopwords/stopwords_it.txt \
+	stopwords/stopwords_nl.txt \
 	stopwords/stopwords_no.txt \
+	stopwords/stopwords_pt.txt \
+	stopwords/stopwords_ro.txt \
+	stopwords/stopwords_ru.txt \
 	stopwords/stopwords_sv.txt
 
 BUILT_SOURCES = word-boundary-data.c word-break-data.c
diff --git a/src/lib-fts/stopwords/stopwords_da.txt b/src/lib-fts/stopwords/stopwords_da.txt
new file mode 100644
index 00000000000..c4cfd9cf04a
--- /dev/null
+++ b/src/lib-fts/stopwords/stopwords_da.txt
@@ -0,0 +1,109 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/danish/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ |  - Encoding was converted to UTF-8.
+ |  - This notice was added.
+ |
+ 
+ | A Danish stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+ | This is a ranked list (commonest to rarest) of stopwords derived from
+ | a large text sample.
+
+
+og           | and
+i            | in
+jeg          | I
+det          | that (dem. pronoun)/it (pers. pronoun)
+at           | that (in front of a sentence)/to (with infinitive)
+en           | a/an
+den          | it (pers. pronoun)/that (dem. pronoun)
+til          | to/at/for/until/against/by/of/into, more
+er           | present tense of "to be"
+som          | who, as
+på           | on/upon/in/on/at/to/after/of/with/for, on
+de           | they
+med          | with/by/in, along
+han          | he
+af           | of/by/from/off/for/in/with/on, off
+for          | at/for/to/from/by/of/ago, in front/before, because
+ikke         | not
+der          | who/which, there/those
+var          | past tense of "to be"
+mig          | me/myself
+sig          | oneself/himself/herself/itself/themselves
+men          | but
+et           | a/an/one, one (number), someone/somebody/one
+har          | present tense of "to have"
+om           | round/about/for/in/a, about/around/down, if
+vi           | we
+min          | my
+havde        | past tense of "to have"
+ham          | him
+hun          | she
+nu           | now
+over         | over/above/across/by/beyond/past/on/about, over/past
+da           | then, when/as/since
+fra          | from/off/since, off, since
+du           | you
+ud           | out
+sin          | his/her/its/one's
+dem          | them
+os           | us/ourselves
+op           | up
+man          | you/one
+hans         | his
+hvor         | where
+eller        | or
+hvad         | what
+skal         | must/shall etc.
+selv         | myself/youself/herself/ourselves etc., even
+her          | here
+alle         | all/everyone/everybody etc.
+vil          | will (verb)
+blev         | past tense of "to stay/to remain/to get/to become"
+kunne        | could
+ind          | in
+når          | when
+være         | present tense of "to be"
+dog          | however/yet/after all
+noget        | something
+ville        | would
+jo           | you know/you see (adv), yes
+deres        | their/theirs
+efter        | after/behind/according to/for/by/from, later/afterwards
+ned          | down
+skulle       | should
+denne        | this
+end          | than
+dette        | this
+mit          | my/mine
+også         | also
+under        | under/beneath/below/during, below/underneath
+have         | have
+dig          | you
+anden        | other
+hende        | her
+mine         | my
+alt          | everything
+meget        | much/very, plenty of
+sit          | his, her, its, one's
+sine         | his, her, its, one's
+vor          | our
+mod          | against
+disse        | these
+hvis         | if
+din          | your/yours
+nogle        | some
+hos          | by/at
+blive        | be/become
+mange        | many
+ad           | by/through
+bliver       | present tense of "to be/to become"
+hendes       | her/hers
+været        | be
+thi          | for (conj)
+jer          | you
+sådan        | such, like this/like that
diff --git a/src/lib-fts/stopwords/stopwords_de.txt b/src/lib-fts/stopwords/stopwords_de.txt
new file mode 100644
index 00000000000..786284d7a58
--- /dev/null
+++ b/src/lib-fts/stopwords/stopwords_de.txt
@@ -0,0 +1,293 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/german/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ |  - Encoding was converted to UTF-8.
+ |  - This notice was added.
+ |
+
+ | A German stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+ | The number of forms in this list is reduced significantly by passing it
+ | through the German stemmer.
+
+
+aber           |  but
+
+alle           |  all
+allem
+allen
+aller
+alles
+
+als            |  than, as
+also           |  so
+am             |  an + dem
+an             |  at
+
+ander          |  other
+andere
+anderem
+anderen
+anderer
+anderes
+anderm
+andern
+anderr
+anders
+
+auch           |  also
+auf            |  on
+aus            |  out of
+bei            |  by
+bin            |  am
+bis            |  until
+bist           |  art
+da             |  there
+damit          |  with it
+dann           |  then
+
+der            |  the
+den
+des
+dem
+die
+das
+
+daß            |  that
+
+derselbe       |  the same
+derselben
+denselben
+desselben
+demselben
+dieselbe
+dieselben
+dasselbe
+
+dazu           |  to that
+
+dein           |  thy
+deine
+deinem
+deinen
+deiner
+deines
+
+denn           |  because
+
+derer          |  of those
+dessen         |  of him
+
+dich           |  thee
+dir            |  to thee
+du             |  thou
+
+dies           |  this
+diese
+diesem
+diesen
+dieser
+dieses
+
+
+doch           |  (several meanings)
+dort           |  (over) there
+
+
+durch          |  through
+
+ein            |  a
+eine
+einem
+einen
+einer
+eines
+
+einig          |  some
+einige
+einigem
+einigen
+einiger
+einiges
+
+einmal         |  once
+
+er             |  he
+ihn            |  him
+ihm            |  to him
+
+es             |  it
+etwas          |  something
+
+euer           |  your
+eure
+eurem
+euren
+eurer
+eures
+
+für            |  for
+gegen          |  towards
+gewesen        |  p.p. of sein
+hab            |  have
+habe           |  have
+haben          |  have
+hat            |  has
+hatte          |  had
+hatten         |  had
+hier           |  here
+hin            |  there
+hinter         |  behind
+
+ich            |  I
+mich           |  me
+mir            |  to me
+
+
+ihr            |  you, to her
+ihre
+ihrem
+ihren
+ihrer
+ihres
+euch           |  to you
+
+im             |  in + dem
+in             |  in
+indem          |  while
+ins            |  in + das
+ist            |  is
+
+jede           |  each, every
+jedem
+jeden
+jeder
+jedes
+
+jene           |  that
+jenem
+jenen
+jener
+jenes
+
+jetzt          |  now
+kann           |  can
+
+kein           |  no
+keine
+keinem
+keinen
+keiner
+keines
+
+können         |  can
+könnte         |  could
+machen         |  do
+man            |  one
+
+manche         |  some, many a
+manchem
+manchen
+mancher
+manches
+
+mein           |  my
+meine
+meinem
+meinen
+meiner
+meines
+
+mit            |  with
+muss           |  must
+musste         |  had to
+nach           |  to(wards)
+nicht          |  not
+nichts         |  nothing
+noch           |  still, yet
+nun            |  now
+nur            |  only
+ob             |  whether
+oder           |  or
+ohne           |  without
+sehr           |  very
+
+sein           |  his
+seine
+seinem
+seinen
+seiner
+seines
+
+selbst         |  self
+sich           |  herself
+
+sie            |  they, she
+ihnen          |  to them
+
+sind           |  are
+so             |  so
+
+solche         |  such
+solchem
+solchen
+solcher
+solches
+
+soll           |  shall
+sollte         |  should
+sondern        |  but
+sonst          |  else
+über           |  over
+um             |  about, around
+und            |  and
+
+uns            |  us
+unse
+unsem
+unsen
+unser
+unses
+
+unter          |  under
+viel           |  much
+vom            |  von + dem
+von            |  from
+vor            |  before
+während        |  while
+war            |  was
+waren          |  were
+warst          |  wast
+was            |  what
+weg            |  away, off
+weil           |  because
+weiter         |  further
+
+welche         |  which
+welchem
+welchen
+welcher
+welches
+
+wenn           |  when
+werde          |  will
+werden         |  will
+wie            |  how
+wieder         |  again
+will           |  want
+wir            |  we
+wird           |  will
+wirst          |  willst
+wo             |  where
+wollen         |  want
+wollte         |  wanted
+würde          |  would
+würden         |  would
+zu             |  to
+zum            |  zu + dem
+zur            |  zu + der
+zwar           |  indeed
+zwischen       |  between
+
diff --git a/src/lib-fts/stopwords/stopwords_es.txt b/src/lib-fts/stopwords/stopwords_es.txt
new file mode 100644
index 00000000000..ed9fdca9c2f
--- /dev/null
+++ b/src/lib-fts/stopwords/stopwords_es.txt
@@ -0,0 +1,355 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/spanish/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ |  - Encoding was converted to UTF-8.
+ |  - This notice was added.
+ |
+
+ | A Spanish stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+
+ | The following is a ranked list (commonest to rarest) of stopwords
+ | deriving from a large sample of text.
+
+ | Extra words have been added at the end.
+
+de             |  from, of
+la             |  the, her
+que            |  who, that
+el             |  the
+en             |  in
+y              |  and
+a              |  to
+los            |  the, them
+del            |  de + el
+se             |  himself, from him etc
+las            |  the, them
+por            |  for, by, etc
+un             |  a
+para           |  for
+con            |  with
+no             |  no
+una            |  a
+su             |  his, her
+al             |  a + el
+  | es         from SER
+lo             |  him
+como           |  how
+más            |  more
+pero           |  pero
+sus            |  su plural
+le             |  to him, her
+ya             |  already
+o              |  or
+  | fue        from SER
+este           |  this
+  | ha         from HABER
+sí             |  himself etc
+porque         |  because
+esta           |  this
+  | son        from SER
+entre          |  between
+  | está     from ESTAR
+cuando         |  when
+muy            |  very
+sin            |  without
+sobre          |  on
+  | ser        from SER
+  | tiene      from TENER
+también        |  also
+me             |  me
+hasta          |  until
+hay            |  there is/are
+donde          |  where
+  | han        from HABER
+quien          |  whom, that
+  | están      from ESTAR
+  | estado     from ESTAR
+desde          |  from
+todo           |  all
+nos            |  us
+durante        |  during
+  | estados    from ESTAR
+todos          |  all
+uno            |  a
+les            |  to them
+ni             |  nor
+contra         |  against
+otros          |  other
+  | fueron     from SER
+ese            |  that
+eso            |  that
+  | había      from HABER
+ante           |  before
+ellos          |  they
+e              |  and (variant of y)
+esto           |  this
+mí             |  me
+antes          |  before
+algunos        |  some
+qué            |  what?
+unos           |  a
+yo             |  I
+otro           |  other
+otras          |  other
+otra           |  other
+él             |  he
+tanto          |  so much, many
+esa            |  that
+estos          |  these
+mucho          |  much, many
+quienes        |  who
+nada           |  nothing
+muchos         |  many
+cual           |  who
+  | sea        from SER
+poco           |  few
+ella           |  she
+estar          |  to be
+  | haber      from HABER
+estas          |  these
+  | estaba     from ESTAR
+  | estamos    from ESTAR
+algunas        |  some
+algo           |  something
+nosotros       |  we
+
+      | other forms
+
+mi             |  me
+mis            |  mi plural
+tú             |  thou
+te             |  thee
+ti             |  thee
+tu             |  thy
+tus            |  tu plural
+ellas          |  they
+nosotras       |  we
+vosotros       |  you
+vosotras       |  you
+os             |  you
+mío            |  mine
+mía            |
+míos           |
+mías           |
+tuyo           |  thine
+tuya           |
+tuyos          |
+tuyas          |
+suyo           |  his, hers, theirs
+suya           |
+suyos          |
+suyas          |
+nuestro        |  ours
+nuestra        |
+nuestros       |
+nuestras       |
+vuestro        |  yours
+vuestra        |
+vuestros       |
+vuestras       |
+esos           |  those
+esas           |  those
+
+               | forms of estar, to be (not including the infinitive):
+estoy
+estás
+está
+estamos
+estáis
+están
+esté
+estés
+estemos
+estéis
+estén
+estaré
+estarás
+estará
+estaremos
+estaréis
+estarán
+estaría
+estarías
+estaríamos
+estaríais
+estarían
+estaba
+estabas
+estábamos
+estabais
+estaban
+estuve
+estuviste
+estuvo
+estuvimos
+estuvisteis
+estuvieron
+estuviera
+estuvieras
+estuviéramos
+estuvierais
+estuvieran
+estuviese
+estuvieses
+estuviésemos
+estuvieseis
+estuviesen
+estando
+estado
+estada
+estados
+estadas
+estad
+
+               | forms of haber, to have (not including the infinitive):
+he
+has
+ha
+hemos
+habéis
+han
+haya
+hayas
+hayamos
+hayáis
+hayan
+habré
+habrás
+habrá
+habremos
+habréis
+habrán
+habría
+habrías
+habríamos
+habríais
+habrían
+había
+habías
+habíamos
+habíais
+habían
+hube
+hubiste
+hubo
+hubimos
+hubisteis
+hubieron
+hubiera
+hubieras
+hubiéramos
+hubierais
+hubieran
+hubiese
+hubieses
+hubiésemos
+hubieseis
+hubiesen
+habiendo
+habido
+habida
+habidos
+habidas
+
+               | forms of ser, to be (not including the infinitive):
+soy
+eres
+es
+somos
+sois
+son
+sea
+seas
+seamos
+seáis
+sean
+seré
+serás
+será
+seremos
+seréis
+serán
+sería
+serías
+seríamos
+seríais
+serían
+era
+eras
+éramos
+erais
+eran
+fui
+fuiste
+fue
+fuimos
+fuisteis
+fueron
+fuera
+fueras
+fuéramos
+fuerais
+fueran
+fuese
+fueses
+fuésemos
+fueseis
+fuesen
+siendo
+sido
+  |  sed also means 'thirst'
+
+               | forms of tener, to have (not including the infinitive):
+tengo
+tienes
+tiene
+tenemos
+tenéis
+tienen
+tenga
+tengas
+tengamos
+tengáis
+tengan
+tendré
+tendrás
+tendrá
+tendremos
+tendréis
+tendrán
+tendría
+tendrías
+tendríamos
+tendríais
+tendrían
+tenía
+tenías
+teníamos
+teníais
+tenían
+tuve
+tuviste
+tuvo
+tuvimos
+tuvisteis
+tuvieron
+tuviera
+tuvieras
+tuviéramos
+tuvierais
+tuvieran
+tuviese
+tuvieses
+tuviésemos
+tuvieseis
+tuviesen
+teniendo
+tenido
+tenida
+tenidos
+tenidas
+tened
+
diff --git a/src/lib-fts/stopwords/stopwords_it.txt b/src/lib-fts/stopwords/stopwords_it.txt
new file mode 100644
index 00000000000..a9e5f676344
--- /dev/null
+++ b/src/lib-fts/stopwords/stopwords_it.txt
@@ -0,0 +1,302 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/italian/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ |  - Encoding was converted to UTF-8.
+ |  - This notice was added.
+ |
+
+ | An Italian stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+ad             |  a (to) before vowel
+al             |  a + il
+allo           |  a + lo
+ai             |  a + i
+agli           |  a + gli
+all            |  a + l'
+agl            |  a + gl'
+alla           |  a + la
+alle           |  a + le
+con            |  with
+col            |  con + il
+coi            |  con + i (forms collo, cogli etc are now very rare)
+da             |  from
+dal            |  da + il
+dallo          |  da + lo
+dai            |  da + i
+dagli          |  da + gli
+dall           |  da + l'
+dagl           |  da + gll'
+dalla          |  da + la
+dalle          |  da + le
+di             |  of
+del            |  di + il
+dello          |  di + lo
+dei            |  di + i
+degli          |  di + gli
+dell           |  di + l'
+degl           |  di + gl'
+della          |  di + la
+delle          |  di + le
+in             |  in
+nel            |  in + el
+nello          |  in + lo
+nei            |  in + i
+negli          |  in + gli
+nell           |  in + l'
+negl           |  in + gl'
+nella          |  in + la
+nelle          |  in + le
+su             |  on
+sul            |  su + il
+sullo          |  su + lo
+sui            |  su + i
+sugli          |  su + gli
+sull           |  su + l'
+sugl           |  su + gl'
+sulla          |  su + la
+sulle          |  su + le
+per            |  through, by
+tra            |  among
+contro         |  against
+io             |  I
+tu             |  thou
+lui            |  he
+lei            |  she
+noi            |  we
+voi            |  you
+loro           |  they
+mio            |  my
+mia            |
+miei           |
+mie            |
+tuo            |
+tua            |
+tuoi           |  thy
+tue            |
+suo            |
+sua            |
+suoi           |  his, her
+sue            |
+nostro         |  our
+nostra         |
+nostri         |
+nostre         |
+vostro         |  your
+vostra         |
+vostri         |
+vostre         |
+mi             |  me
+ti             |  thee
+ci             |  us, there
+vi             |  you, there
+lo             |  him, the
+la             |  her, the
+li             |  them
+le             |  them, the
+gli            |  to him, the
+ne             |  from there etc
+il             |  the
+un             |  a
+uno            |  a
+una            |  a
+ma             |  but
+ed             |  and
+se             |  if
+perché         |  why, because
+anche          |  also
+come           |  how
+dov            |  where (as dov')
+dove           |  where
+che            |  who, that
+chi            |  who
+cui            |  whom
+non            |  not
+più            |  more
+quale          |  who, that
+quanto         |  how much
+quanti         |
+quanta         |
+quante         |
+quello         |  that
+quelli         |
+quella         |
+quelle         |
+questo         |  this
+questi         |
+questa         |
+queste         |
+si             |  yes
+tutto          |  all
+tutti          |  all
+
+               |  single letter forms:
+
+a              |  at
+c              |  as c' for ce or ci
+e              |  and
+i              |  the
+l              |  as l'
+o              |  or
+
+               | forms of avere, to have (not including the infinitive):
+
+ho
+hai
+ha
+abbiamo
+avete
+hanno
+abbia
+abbiate
+abbiano
+avrò
+avrai
+avrà
+avremo
+avrete
+avranno
+avrei
+avresti
+avrebbe
+avremmo
+avreste
+avrebbero
+avevo
+avevi
+aveva
+avevamo
+avevate
+avevano
+ebbi
+avesti
+ebbe
+avemmo
+aveste
+ebbero
+avessi
+avesse
+avessimo
+avessero
+avendo
+avuto
+avuta
+avuti
+avute
+
+               | forms of essere, to be (not including the infinitive):
+sono
+sei
+è
+siamo
+siete
+sia
+siate
+siano
+sarò
+sarai
+sarà
+saremo
+sarete
+saranno
+sarei
+saresti
+sarebbe
+saremmo
+sareste
+sarebbero
+ero
+eri
+era
+eravamo
+eravate
+erano
+fui
+fosti
+fu
+fummo
+foste
+furono
+fossi
+fosse
+fossimo
+fossero
+essendo
+
+               | forms of fare, to do (not including the infinitive, fa, fat-):
+faccio
+fai
+facciamo
+fanno
+faccia
+facciate
+facciano
+farò
+farai
+farà
+faremo
+farete
+faranno
+farei
+faresti
+farebbe
+faremmo
+fareste
+farebbero
+facevo
+facevi
+faceva
+facevamo
+facevate
+facevano
+feci
+facesti
+fece
+facemmo
+faceste
+fecero
+facessi
+facesse
+facessimo
+facessero
+facendo
+
+               | forms of stare, to be (not including the infinitive):
+sto
+stai
+sta
+stiamo
+stanno
+stia
+stiate
+stiano
+starò
+starai
+starà
+staremo
+starete
+staranno
+starei
+staresti
+starebbe
+staremmo
+stareste
+starebbero
+stavo
+stavi
+stava
+stavamo
+stavate
+stavano
+stetti
+stesti
+stette
+stemmo
+steste
+stettero
+stessi
+stesse
+stessimo
+stessero
+stando
diff --git a/src/lib-fts/stopwords/stopwords_nl.txt b/src/lib-fts/stopwords/stopwords_nl.txt
new file mode 100644
index 00000000000..6fc4ce7e359
--- /dev/null
+++ b/src/lib-fts/stopwords/stopwords_nl.txt
@@ -0,0 +1,118 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/dutch/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ |  - Encoding was converted to UTF-8.
+ |  - This notice was added.
+ |
+
+ | A Dutch stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+ | This is a ranked list (commonest to rarest) of stopwords derived from
+ | a large sample of Dutch text.
+
+ | Dutch stop words frequently exhibit homonym clashes. These are indicated
+ | clearly below.
+
+de             |  the
+en             |  and
+van            |  of, from
+ik             |  I, the ego
+te             |  (1) chez, at etc, (2) to, (3) too
+dat            |  that, which
+die            |  that, those, who, which
+in             |  in, inside
+een            |  a, an, one
+hij            |  he
+het            |  the, it
+niet           |  not, nothing, naught
+zijn           |  (1) to be, being, (2) his, one's, its
+is             |  is
+was            |  (1) was, past tense of all persons sing. of 'zijn' (to be) (2) wax, (3) the washing, (4) rise of river
+op             |  on, upon, at, in, up, used up
+aan            |  on, upon, to (as dative)
+met            |  with, by
+als            |  like, such as, when
+voor           |  (1) before, in front of, (2) furrow
+had            |  had, past tense all persons sing. of 'hebben' (have)
+er             |  there
+maar           |  but, only
+om             |  round, about, for etc
+hem            |  him
+dan            |  then
+zou            |  should/would, past tense all persons sing. of 'zullen'
+of             |  or, whether, if
+wat            |  what, something, anything
+mijn           |  possessive and noun 'mine'
+men            |  people, 'one'
+dit            |  this
+zo             |  so, thus, in this way
+door           |  through by
+over           |  over, across
+ze             |  she, her, they, them
+zich           |  oneself
+bij            |  (1) a bee, (2) by, near, at
+ook            |  also, too
+tot            |  till, until
+je             |  you
+mij            |  me
+uit            |  out of, from
+der            |  Old Dutch form of 'van der' still found in surnames
+daar           |  (1) there, (2) because
+haar           |  (1) her, their, them, (2) hair
+naar           |  (1) unpleasant, unwell etc, (2) towards, (3) as
+heb            |  present first person sing. of 'to have'
+hoe            |  how, why
+heeft          |  present third person sing. of 'to have'
+hebben         |  'to have' and various parts thereof
+deze           |  this
+u              |  you
+want           |  (1) for, (2) mitten, (3) rigging
+nog            |  yet, still
+zal            |  'shall', first and third person sing. of verb 'zullen' (will)
+me             |  me
+zij            |  she, they
+nu             |  now
+ge             |  'thou', still used in Belgium and south Netherlands
+geen           |  none
+omdat          |  because
+iets           |  something, somewhat
+worden         |  to become, grow, get
+toch           |  yet, still
+al             |  all, every, each
+waren          |  (1) 'were' (2) to wander, (3) wares, (3)
+veel           |  much, many
+meer           |  (1) more, (2) lake
+doen           |  to do, to make
+toen           |  then, when
+moet           |  noun 'spot/mote' and present form of 'to must'
+ben            |  (1) am, (2) 'are' in interrogative second person singular of 'to be'
+zonder         |  without
+kan            |  noun 'can' and present form of 'to be able'
+hun            |  their, them
+dus            |  so, consequently
+alles          |  all, everything, anything
+onder          |  under, beneath
+ja             |  yes, of course
+eens           |  once, one day
+hier           |  here
+wie            |  who
+werd           |  imperfect third person sing. of 'become'
+altijd         |  always
+doch           |  yet, but etc
+wordt          |  present third person sing. of 'become'
+wezen          |  (1) to be, (2) 'been' as in 'been fishing', (3) orphans
+kunnen         |  to be able
+ons            |  us/our
+zelf           |  self
+tegen          |  against, towards, at
+na             |  after, near
+reeds          |  already
+wil            |  (1) present tense of 'want', (2) 'will', noun, (3) fender
+kon            |  could; past tense of 'to be able'
+niets          |  nothing
+uw             |  your
+iemand         |  somebody
+geweest        |  been; past participle of 'be'
+andere         |  other
diff --git a/src/lib-fts/stopwords/stopwords_pt.txt b/src/lib-fts/stopwords/stopwords_pt.txt
new file mode 100644
index 00000000000..37449615ef6
--- /dev/null
+++ b/src/lib-fts/stopwords/stopwords_pt.txt
@@ -0,0 +1,252 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/portuguese/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ |  - Encoding was converted to UTF-8.
+ |  - This notice was added.
+ |
+
+ | A Portuguese stop word list. Comments begin with vertical bar. Each stop
+ | word is at the start of a line.
+
+
+ | The following is a ranked list (commonest to rarest) of stopwords
+ | deriving from a large sample of text.
+
+ | Extra words have been added at the end.
+
+de             |  of, from
+a              |  the; to, at; her
+o              |  the; him
+que            |  who, that
+e              |  and
+do             |  de + o
+da             |  de + a
+em             |  in
+um             |  a
+para           |  for
+  | é          from SER
+com            |  with
+não            |  not, no
+uma            |  a
+os             |  the; them
+no             |  em + o
+se             |  himself etc
+na             |  em + a
+por            |  for
+mais           |  more
+as             |  the; them
+dos            |  de + os
+como           |  as, like
+mas            |  but
+  | foi        from SER
+ao             |  a + o
+ele            |  he
+das            |  de + as
+  | tem        from TER
+à              |  a + a
+seu            |  his
+sua            |  her
+ou             |  or
+  | ser        from SER
+quando         |  when
+muito          |  much
+  | há         from HAV
+nos            |  em + os; us
+já             |  already, now
+  | está       from EST
+eu             |  I
+também         |  also
+só             |  only, just
+pelo           |  per + o
+pela           |  per + a
+até            |  up to
+isso           |  that
+ela            |  he
+entre          |  between
+  | era        from SER
+depois         |  after
+sem            |  without
+mesmo          |  same
+aos            |  a + os
+  | ter        from TER
+seus           |  his
+quem           |  whom
+nas            |  em + as
+me             |  me
+esse           |  that
+eles           |  they
+  | estão      from EST
+você           |  you
+  | tinha      from TER
+  | foram      from SER
+essa           |  that
+num            |  em + um
+nem            |  nor
+suas           |  her
+meu            |  my
+às             |  a + as
+minha          |  my
+  | têm        from TER
+numa           |  em + uma
+pelos          |  per + os
+elas           |  they
+  | havia      from HAV
+  | seja       from SER
+qual           |  which
+  | será       from SER
+nós            |  we
+  | tenho      from TER
+lhe            |  to him, her
+deles          |  of them
+essas          |  those
+esses          |  those
+pelas          |  per + as
+este           |  this
+  | fosse      from SER
+dele           |  of him
+
+ | other words. There are many contractions such as naquele = em+aquele,
+ | mo = me+o, but they are rare.
+ | Indefinite article plural forms are also rare.
+
+tu             |  thou
+te             |  thee
+vocês          |  you (plural)
+vos            |  you
+lhes           |  to them
+meus           |  my
+minhas
+teu            |  thy
+tua
+teus
+tuas
+nosso          | our
+nossa
+nossos
+nossas
+
+dela           |  of her
+delas          |  of them
+
+esta           |  this
+estes          |  these
+estas          |  these
+aquele         |  that
+aquela         |  that
+aqueles        |  those
+aquelas        |  those
+isto           |  this
+aquilo         |  that
+
+               | forms of estar, to be (not including the infinitive):
+estou
+está
+estamos
+estão
+estive
+esteve
+estivemos
+estiveram
+estava
+estávamos
+estavam
+estivera
+estivéramos
+esteja
+estejamos
+estejam
+estivesse
+estivéssemos
+estivessem
+estiver
+estivermos
+estiverem
+
+               | forms of haver, to have (not including the infinitive):
+hei
+há
+havemos
+hão
+houve
+houvemos
+houveram
+houvera
+houvéramos
+haja
+hajamos
+hajam
+houvesse
+houvéssemos
+houvessem
+houver
+houvermos
+houverem
+houverei
+houverá
+houveremos
+houverão
+houveria
+houveríamos
+houveriam
+
+               | forms of ser, to be (not including the infinitive):
+sou
+somos
+são
+era
+éramos
+eram
+fui
+foi
+fomos
+foram
+fora
+fôramos
+seja
+sejamos
+sejam
+fosse
+fôssemos
+fossem
+for
+formos
+forem
+serei
+será
+seremos
+serão
+seria
+seríamos
+seriam
+
+               | forms of ter, to have (not including the infinitive):
+tenho
+tem
+temos
+tém
+tinha
+tínhamos
+tinham
+tive
+teve
+tivemos
+tiveram
+tivera
+tivéramos
+tenha
+tenhamos
+tenham
+tivesse
+tivéssemos
+tivessem
+tiver
+tivermos
+tiverem
+terei
+terá
+teremos
+terão
+teria
+teríamos
+teriam
diff --git a/src/lib-fts/stopwords/stopwords_ro.txt b/src/lib-fts/stopwords/stopwords_ro.txt
new file mode 100644
index 00000000000..4fdee90a5ba
--- /dev/null
+++ b/src/lib-fts/stopwords/stopwords_ro.txt
@@ -0,0 +1,233 @@
+# This file was created by Jacques Savoy and is distributed under the BSD license.
+# See http://members.unine.ch/jacques.savoy/clef/index.html.
+# Also see http://www.opensource.org/licenses/bsd-license.html
+acea
+aceasta
+această
+aceea
+acei
+aceia
+acel
+acela
+acele
+acelea
+acest
+acesta
+aceste
+acestea
+aceşti
+aceştia
+acolo
+acum
+ai
+aia
+aibă
+aici
+al
+ăla
+ale
+alea
+ălea
+altceva
+altcineva
+am
+ar
+are
+aş
+aşadar
+asemenea
+asta
+ăsta
+astăzi
+astea
+ăstea
+ăştia
+asupra
+aţi
+au
+avea
+avem
+aveţi
+azi
+bine
+bucur
+bună
+ca
+că
+căci
+când
+care
+cărei
+căror
+cărui
+cât
+câte
+câţi
+către
+câtva
+ce
+cel
+ceva
+chiar
+cînd
+cine
+cineva
+cît
+cîte
+cîţi
+cîtva
+contra
+cu
+cum
+cumva
+curând
+curînd
+da
+dă
+dacă
+dar
+datorită
+de
+deci
+deja
+deoarece
+departe
+deşi
+din
+dinaintea
+dintr
+dintre
+drept
+după
+ea
+ei
+el
+ele
+eram
+este
+eşti
+eu
+face
+fără
+fi
+fie
+fiecare
+fii
+fim
+fiţi
+iar
+ieri
+îi
+îl
+îmi
+împotriva
+în 
+înainte
+înaintea
+încât
+încît
+încotro
+între
+întrucât
+întrucît
+îţi
+la
+lângă
+le
+li
+lîngă
+lor
+lui
+mă
+mâine
+mea
+mei
+mele
+mereu
+meu
+mi
+mine
+mult
+multă
+mulţi
+ne
+nicăieri
+nici
+nimeni
+nişte
+noastră
+noastre
+noi
+noştri
+nostru
+nu
+ori
+oricând
+oricare
+oricât
+orice
+oricînd
+oricine
+oricît
+oricum
+oriunde
+până
+pe
+pentru
+peste
+pînă
+poate
+pot
+prea
+prima
+primul
+prin
+printr
+sa
+să
+săi
+sale
+sau
+său
+se
+şi
+sînt
+sîntem
+sînteţi
+spre
+sub
+sunt
+suntem
+sunteţi
+ta
+tăi
+tale
+tău
+te
+ţi
+ţie
+tine
+toată
+toate
+tot
+toţi
+totuşi
+tu
+un
+una
+unde
+undeva
+unei
+unele
+uneori
+unor
+vă
+vi
+voastră
+voastre
+voi
+voştri
+vostru
+vouă
+vreo
+vreun
diff --git a/src/lib-fts/stopwords/stopwords_ru.txt b/src/lib-fts/stopwords/stopwords_ru.txt
new file mode 100644
index 00000000000..36b0d9fd277
--- /dev/null
+++ b/src/lib-fts/stopwords/stopwords_ru.txt
@@ -0,0 +1,242 @@
+ | From svn.tartarus.org/snowball/trunk/website/algorithms/russian/stop.txt
+ | This file is distributed under the BSD License.
+ | See http://snowball.tartarus.org/license.php
+ | Also see http://www.opensource.org/licenses/bsd-license.html
+ |  - Encoding was converted to UTF-8.
+ |  - This notice was added.
+ |
+
+ | a russian stop word list. comments begin with vertical bar. each stop
+ | word is at the start of a line.
+
+ | this is a ranked list (commonest to rarest) of stopwords derived from
+ | a large text sample.
+
+ | letter `ё' is translated to `е'.
+
+и              | and
+в              | in/into
+во             | alternative form
+не             | not
+что            | what/that
+он             | he
+на             | on/onto
+я              | i
+с              | from
+со             | alternative form
+как            | how
+а              | milder form of `no' (but)
+то             | conjunction and form of `that'
+все            | all
+она            | she
+так            | so, thus
+его            | him
+но             | but
+да             | yes/and
+ты             | thou
+к              | towards, by
+у              | around, chez
+же             | intensifier particle
+вы             | you
+за             | beyond, behind
+бы             | conditional/subj. particle
+по             | up to, along
+только         | only
+ее             | her
+мне            | to me
+было           | it was
+вот            | here is/are, particle
+от             | away from
+меня           | me
+еще            | still, yet, more
+нет            | no, there isnt/arent
+о              | about
+из             | out of
+ему            | to him
+теперь         | now
+когда          | when
+даже           | even
+ну             | so, well
+вдруг          | suddenly
+ли             | interrogative particle
+если           | if
+уже            | already, but homonym of `narrower'
+или            | or
+ни             | neither
+быть           | to be
+был            | he was
+него           | prepositional form of его
+до             | up to
+вас            | you accusative
+нибудь         | indef. suffix preceded by hyphen
+опять          | again
+уж             | already, but homonym of `adder'
+вам            | to you
+сказал         | he said
+ведь           | particle `after all'
+там            | there
+потом          | then
+себя           | oneself
+ничего         | nothing
+ей             | to her
+может          | usually with `быть' as `maybe'
+они            | they
+тут            | here
+где            | where
+есть           | there is/are
+надо           | got to, must
+ней            | prepositional form of  ей
+для            | for
+мы             | we
+тебя           | thee
+их             | them, their
+чем            | than
+была           | she was
+сам            | self
+чтоб           | in order to
+без            | without
+будто          | as if
+человек        | man, person, one
+чего           | genitive form of `what'
+раз            | once
+тоже           | also
+себе           | to oneself
+под            | beneath
+жизнь          | life
+будет          | will be
+ж              | short form of intensifer particle `же'
+тогда          | then
+кто            | who
+этот           | this
+говорил        | was saying
+того           | genitive form of `that'
+потому         | for that reason
+этого          | genitive form of `this'
+какой          | which
+совсем         | altogether
+ним            | prepositional form of `его', `они'
+здесь          | here
+этом           | prepositional form of `этот'
+один           | one
+почти          | almost
+мой            | my
+тем            | instrumental/dative plural of `тот', `то'
+чтобы          | full form of `in order that'
+нее            | her (acc.)
+кажется        | it seems
+сейчас         | now
+были           | they were
+куда           | where to
+зачем          | why
+сказать        | to say
+всех           | all (acc., gen. preposn. plural)
+никогда        | never
+сегодня        | today
+можно          | possible, one can
+при            | by
+наконец        | finally
+два            | two
+об             | alternative form of `о', about
+другой         | another
+хоть           | even
+после          | after
+над            | above
+больше         | more
+тот            | that one (masc.)
+через          | across, in
+эти            | these
+нас            | us
+про            | about
+всего          | in all, only, of all
+них            | prepositional form of `они' (they)
+какая          | which, feminine
+много          | lots
+разве          | interrogative particle
+сказала        | she said
+три            | three
+эту            | this, acc. fem. sing.
+моя            | my, feminine
+впрочем        | moreover, besides
+хорошо         | good
+свою           | ones own, acc. fem. sing.
+этой           | oblique form of `эта', fem. `this'
+перед          | in front of
+иногда         | sometimes
+лучше          | better
+чуть           | a little
+том            | preposn. form of `that one'
+нельзя         | one must not
+такой          | such a one
+им             | to them
+более          | more
+всегда         | always
+конечно        | of course
+всю            | acc. fem. sing of `all'
+между          | between
+
+
+  | b: some paradigms
+  |
+  | personal pronouns
+  |
+  | я  меня  мне  мной  [мною]
+  | ты  тебя  тебе  тобой  [тобою]
+  | он  его  ему  им  [него, нему, ним]
+  | она  ее  эи  ею  [нее, нэи, нею]
+  | оно  его  ему  им  [него, нему, ним]
+  |
+  | мы  нас  нам  нами
+  | вы  вас  вам  вами
+  | они  их  им  ими  [них, ним, ними]
+  |
+  |   себя  себе  собой   [собою]
+  |
+  | demonstrative pronouns: этот (this), тот (that)
+  |
+  | этот  эта  это  эти
+  | этого  эты  это  эти
+  | этого  этой  этого  этих
+  | этому  этой  этому  этим
+  | этим  этой  этим  [этою]  этими
+  | этом  этой  этом  этих
+  |
+  | тот  та  то  те
+  | того  ту  то  те
+  | того  той  того  тех
+  | тому  той  тому  тем
+  | тем  той  тем  [тою]  теми
+  | том  той  том  тех
+  |
+  | determinative pronouns
+  |
+  | (a) весь (all)
+  |
+  | весь  вся  все  все
+  | всего  всю  все  все
+  | всего  всей  всего  всех
+  | всему  всей  всему  всем
+  | всем  всей  всем  [всею]  всеми
+  | всем  всей  всем  всех
+  |
+  | (b) сам (himself etc)
+  |
+  | сам  сама  само  сами
+  | самого саму  само  самих
+  | самого самой самого  самих
+  | самому самой самому  самим
+  | самим  самой  самим  [самою]  самими
+  | самом самой самом  самих
+  |
+  | stems of verbs `to be', `to have', `to do' and modal
+  |
+  | быть  бы  буд  быв  есть  суть
+  | име
+  | дел
+  | мог   мож  мочь
+  | уме
+  | хоч  хот
+  | долж
+  | можн
+  | нужн
+  | нельзя
+

From f6e14400c10dfc148775623c1547c4e6246d397e Mon Sep 17 00:00:00 2001
From: Teemu Huovila <teemu.huovila@dovecot.fi>
Date: Wed, 1 Jun 2016 12:24:57 +0300
Subject: [PATCH 222/450] lib-fts: Improved stopword file reading.

The reading tries to be a little bit stricter now. Only stopwords at the
start of a new line are accepted now. Changed fi stopwords accordingly.
Also removed superfluous stack allocation in parsing.
---
 src/lib-fts/Makefile.am                       |    3 +-
 src/lib-fts/fts-filter-stopwords.c            |   30 +-
 src/lib-fts/stopwords/stopwords_fi.txt        |  207 +-
 src/lib-fts/stopwords/stopwords_malformed.txt | 2866 +++++++++++++++++
 src/lib-fts/test-fts-filter.c                 |   17 +
 5 files changed, 3088 insertions(+), 35 deletions(-)
 create mode 100644 src/lib-fts/stopwords/stopwords_malformed.txt

diff --git a/src/lib-fts/Makefile.am b/src/lib-fts/Makefile.am
index 4eb3f49e939..1219ddf82a9 100644
--- a/src/lib-fts/Makefile.am
+++ b/src/lib-fts/Makefile.am
@@ -38,7 +38,8 @@ EXTRA_DIST = \
 	word-properties.pl \
 	WordBreakProperty.txt \
 	word-boundary-data.c \
-	word-break-data.c
+	word-break-data.c \
+	stopwords/stopwords_malformed.txt
 
 WordBreakProperty.txt:
 	test -f WordBreakProperty.txt || wget http://www.unicode.org/Public/UNIDATA/auxiliary/WordBreakProperty.txt
diff --git a/src/lib-fts/fts-filter-stopwords.c b/src/lib-fts/fts-filter-stopwords.c
index 2d6df6964f9..8b7a1067a55 100644
--- a/src/lib-fts/fts-filter-stopwords.c
+++ b/src/lib-fts/fts-filter-stopwords.c
@@ -11,8 +11,8 @@
 
 #define STOPWORDS_FILE_FORMAT "%s/stopwords_%s.txt"
 
-#define STOPWORDS_COMMENT_CHAR1 '|'
-#define STOPWORDS_COMMENT_CHAR2 '#'
+#define STOPWORDS_CUTCHARS "|#\t "
+#define STOPWORDS_DISALLOWED_CHARS "/\\<>.,\":()\t\n\r"
 
 struct fts_filter_stopwords {
 	struct fts_filter filter;
@@ -26,29 +26,33 @@ static int fts_filter_stopwords_read_list(struct fts_filter_stopwords *filter,
 					  const char **error_r)
 {
 	struct istream *input;
-	const char *line, **words, *path;
+	const char *line, *word, *path;
 	int ret = 0;
+	size_t len;
 
 	path = t_strdup_printf(STOPWORDS_FILE_FORMAT,
 			       filter->stopwords_dir, filter->lang->name);
 
 	input = i_stream_create_file(path, IO_BLOCK_SIZE);
-	while ((line = i_stream_read_next_line(input)) != NULL) T_BEGIN {
-		line = t_strcut(line, STOPWORDS_COMMENT_CHAR1);
-		line = t_strcut(line, STOPWORDS_COMMENT_CHAR2);
-
-		words = t_strsplit_spaces(line, " \t");
-		for (; *words != NULL; words++) {
-			const char *word = p_strdup(filter->pool, *words);
-			hash_table_insert(filter->stopwords, word, word);
-		}
-	} T_END;
+	while ((line = i_stream_read_next_line(input)) != NULL) {
+		len = strcspn(line, STOPWORDS_CUTCHARS);
+		if (len == 0)
+			continue;
+		if (strcspn(line, STOPWORDS_DISALLOWED_CHARS) < len)
+			continue;
+		word = p_strndup(filter->pool, line, len);
+		hash_table_insert(filter->stopwords, word, word);
+	}
 
 	if (input->stream_errno != 0) {
 		*error_r = t_strdup_printf("Failed to read stopword list %s: %s",
 					   path, i_stream_get_error(input));
 		ret = -1;
 	}
+
+	if (ret == 0 && hash_table_count(filter->stopwords) == 0)
+		i_warning("Stopwords list \"%s\" seems empty. Is the file correctly formatted?", path);
+
 	i_stream_destroy(&input);
 	return ret;
 }
diff --git a/src/lib-fts/stopwords/stopwords_fi.txt b/src/lib-fts/stopwords/stopwords_fi.txt
index addad798c4b..95c74f256b8 100644
--- a/src/lib-fts/stopwords/stopwords_fi.txt
+++ b/src/lib-fts/stopwords/stopwords_fi.txt
@@ -38,28 +38,193 @@ emme
 ette
 eivät
 
+|Words below in the following cases:
 |Nom   Gen    Acc    Part   Iness   Elat    Illat  Adess   Ablat   Allat   Ess    Trans
-minä   minun  minut  minua  minussa minusta minuun minulla minulta minulle               | I
-sinä   sinun  sinut  sinua  sinussa sinusta sinuun sinulla sinulta sinulle               | you
-hän    hänen  hänet  häntä  hänessä hänestä häneen hänellä häneltä hänelle               | he she
-me     meidän meidät meitä  meissä  meistä  meihin meillä  meiltä  meille                | we
-te     teidän teidät teitä  teissä  teistä  teihin teillä  teiltä  teille                | you
-he     heidän heidät heitä  heissä  heistä  heihin heillä  heiltä  heille                | they
-
-tämä   tämän         tätä   tässä   tästä   tähän  tallä   tältä   tälle   tänä   täksi  | this
-tuo    tuon          tuotä  tuossa  tuosta  tuohon tuolla  tuolta  tuolle  tuona  tuoksi | that
-se     sen           sitä   siinä   siitä   siihen sillä   siltä   sille   sinä   siksi  | it
-nämä   näiden        näitä  näissä  näistä  näihin näillä  näiltä  näille  näinä  näiksi | these
-nuo    noiden        noita  noissa  noista  noihin noilla  noilta  noille  noina  noiksi | those
-ne     niiden        niitä  niissä  niistä  niihin niillä  niiltä  niille  niinä  niiksi | they
-
-kuka   kenen kenet   ketä   kenessä kenestä keneen kenellä keneltä kenelle kenenä keneksi| who
-ketkä  keiden ketkä  keitä  keissä  keistä  keihin keillä  keiltä  keille  keinä  keiksi | (pl)
-mikä   minkä minkä   mitä   missä   mistä   mihin  millä   miltä   mille   minä   miksi  | which what
-mitkä                                                                                    | (pl)
-
-joka   jonka         jota   jossa   josta   johon  jolla   jolta   jolle   jona   joksi  | who which
-jotka  joiden        joita  joissa  joista  joihin joilla  joilta  joille  joina  joiksi | (pl)
+minä | I
+minun | I
+minut | I
+minua | I
+minussa | I
+minusta | I
+minuun | I
+minulla | I
+minulta | I
+minulle | I
+sinä | you
+sinun | you
+sinut | you
+sinua | you
+sinussa | you
+sinusta | you
+sinuun | you
+sinulla | you
+sinulta | you
+sinulle | you
+hän | he she
+hänen | he she
+hänet | he she
+häntä | he she
+hänessä | he she
+hänestä | he she
+häneen | he she
+hänellä | he she
+häneltä | he she
+hänelle | he she
+me | we
+meidän | we
+meidät | we
+meitä | we
+meissä | we
+meistä | we
+meihin | we
+meillä | we
+meiltä | we
+meille | we
+te | you
+teidän | you
+teidät | you
+teitä | you
+teissä | you
+teistä | you
+teihin | you
+teillä | you
+teiltä | you
+teille | you
+he | they
+heidän | they
+heidät | they
+heitä | they
+heissä | they
+heistä | they
+heihin | they
+heillä | they
+heiltä | they
+heille | they
+tämä | this
+tämän | this
+tätä | this
+tässä | this
+tästä | this
+tähän | this
+tallä | this
+tältä | this
+tälle | this
+tänä | this
+täksi | this
+tuo | that
+tuon | that
+tuotä | that
+tuossa | that
+tuosta | that
+tuohon | that
+tuolla | that
+tuolta | that
+tuolle | that
+tuona | that
+tuoksi | that
+se | it
+sen | it
+sitä | it
+siinä | it
+siitä | it
+siihen | it
+sillä | it
+siltä | it
+sille | it
+sinä | it
+siksi | it
+nämä | these
+näiden | these
+näitä | these
+näissä | these
+näistä | these
+näihin | these
+näillä | these
+näiltä | these
+näille | these
+näinä | these
+näiksi | these
+nuo | those
+noiden | those
+noita | those
+noissa | those
+noista | those
+noihin | those
+noilla | those
+noilta | those
+noille | those
+noina | those
+noiksi | those
+ne | they
+niiden | they
+niitä | they
+niissä | they
+niistä | they
+niihin | they
+niillä | they
+niiltä | they
+niille | they
+niinä | they
+niiksi | they
+kuka | who
+kenen | who
+kenet | who
+ketä | who
+kenessä | who
+kenestä | who
+keneen | who
+kenellä | who
+keneltä | who
+kenelle | who
+kenenä | who
+keneksi | who
+ketkä | who (pl)
+keiden | who (pl)
+ketkä | who (pl)
+keitä | who (pl)
+keissä | who (pl)
+keistä | who (pl)
+keihin | who (pl)
+keillä | who (pl)
+keiltä | who (pl)
+keille | who (pl)
+keinä | who (pl)
+keiksi | who (pl)
+mikä | which what
+minkä | which what
+minkä | which what
+mitä | which what
+missä | which what
+mistä | which what
+mihin | which what
+millä | which what
+miltä | which what
+mille | which what
+minä | which what
+miksi | which what
+mitkä | which what (pl)
+joka | who which
+jonka | who which
+jota | who which
+jossa | who which
+josta | who which
+johon | who which
+jolla | who which
+jolta | who which
+jolle | who which
+jona | who which
+joksi | who which
+jotka | who which  (pl)
+joiden | who which  (pl)
+joita | who which  (pl)
+joissa | who which  (pl)
+joista | who which  (pl)
+joihin | who which  (pl)
+joilla | who which  (pl)
+joilta | who which  (pl)
+joille | who which  (pl)
+joina | who which  (pl)
+joiksi | who which  (pl)
 
 | conjunctions
 
diff --git a/src/lib-fts/stopwords/stopwords_malformed.txt b/src/lib-fts/stopwords/stopwords_malformed.txt
new file mode 100644
index 00000000000..5899230617a
--- /dev/null
+++ b/src/lib-fts/stopwords/stopwords_malformed.txt
@@ -0,0 +1,2866 @@
+
+
+
+
+<!DOCTYPE html>
+<html lang="en" class="">
+  <head prefix="og: http://ogp.me/ns# fb: http://ogp.me/ns/fb# object: http://ogp.me/ns/object# article: http://ogp.me/ns/article# profile: http://ogp.me/ns/profile#">
+    <meta charset='utf-8'>
+
+    <link crossorigin="anonymous" href="https://assets-cdn.github.com/assets/github-5a3552c77198dc1a3f6ff70409e1b589ad5438e7449023c3b68ca5f49e639fb0.css" media="all" rel="stylesheet" />
+    <link crossorigin="anonymous" href="https://assets-cdn.github.com/assets/github2-e7d851ea8a5986690487ea49678059eb4c52bb84e2b4b13d9cb7b93f6115e21c.css" media="all" rel="stylesheet" />
+    
+    
+    
+
+    <link as="script" href="https://assets-cdn.github.com/assets/frameworks-96e29b697be2d5e4c062e40ce2dab35f90b4cd378cfbbcd6dd51a3f280e7d426.js" rel="preload" />
+    <link as="script" href="https://assets-cdn.github.com/assets/github-061f27987895ca9e650b5c852acbea14f7b6e360f8d0a2bf2e5692911aa3926c.js" rel="preload" />
+
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta http-equiv="Content-Language" content="en">
+    <meta name="viewport" content="width=1020">
+    
+    
+    <title>lucene-solr/stopwords_de.txt at master · apache/lucene-solr · GitHub</title>
+    <link rel="search" type="application/opensearchdescription+xml" href="/opensearch.xml" title="GitHub">
+    <link rel="fluid-icon" href="https://github.com/fluidicon.png" title="GitHub">
+    <link rel="apple-touch-icon" href="/apple-touch-icon.png">
+    <link rel="apple-touch-icon" sizes="57x57" href="/apple-touch-icon-57x57.png">
+    <link rel="apple-touch-icon" sizes="60x60" href="/apple-touch-icon-60x60.png">
+    <link rel="apple-touch-icon" sizes="72x72" href="/apple-touch-icon-72x72.png">
+    <link rel="apple-touch-icon" sizes="76x76" href="/apple-touch-icon-76x76.png">
+    <link rel="apple-touch-icon" sizes="114x114" href="/apple-touch-icon-114x114.png">
+    <link rel="apple-touch-icon" sizes="120x120" href="/apple-touch-icon-120x120.png">
+    <link rel="apple-touch-icon" sizes="144x144" href="/apple-touch-icon-144x144.png">
+    <link rel="apple-touch-icon" sizes="152x152" href="/apple-touch-icon-152x152.png">
+    <link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon-180x180.png">
+    <meta property="fb:app_id" content="1401488693436528">
+
+      <meta content="https://avatars1.githubusercontent.com/u/47359?v=3&amp;s=400" name="twitter:image:src" /><meta content="@github" name="twitter:site" /><meta content="summary" name="twitter:card" /><meta content="apache/lucene-solr" name="twitter:title" /><meta content="lucene-solr - Mirror of Apache Lucene + Solr" name="twitter:description" />
+      <meta content="https://avatars1.githubusercontent.com/u/47359?v=3&amp;s=400" property="og:image" /><meta content="GitHub" property="og:site_name" /><meta content="object" property="og:type" /><meta content="apache/lucene-solr" property="og:title" /><meta content="https://github.com/apache/lucene-solr" property="og:url" /><meta content="lucene-solr - Mirror of Apache Lucene + Solr" property="og:description" />
+      <meta name="browser-stats-url" content="https://api.github.com/_private/browser/stats">
+    <meta name="browser-errors-url" content="https://api.github.com/_private/browser/errors">
+    <link rel="assets" href="https://assets-cdn.github.com/">
+    
+    <meta name="pjax-timeout" content="1000">
+    
+
+    <meta name="msapplication-TileImage" content="/windows-tile.png">
+    <meta name="msapplication-TileColor" content="#ffffff">
+    <meta name="selected-link" value="repo_source" data-pjax-transient>
+
+    <meta name="google-site-verification" content="KT5gs8h0wvaagLKAVWq8bbeNwnZZK1r1XQysX3xurLU">
+<meta name="google-site-verification" content="ZzhVyEFwb7w3e0-uOTltm8Jsck2F5StVihD0exw2fsA">
+    <meta name="google-analytics" content="UA-3769691-2">
+
+<meta content="collector.githubapp.com" name="octolytics-host" /><meta content="github" name="octolytics-app-id" /><meta content="D906D48A:6B89:225E586:56D82F8C" name="octolytics-dimension-request_id" />
+<meta content="/&lt;user-name&gt;/&lt;repo-name&gt;/blob/show" data-pjax-transient="true" name="analytics-location" />
+
+
+
+  <meta class="js-ga-set" name="dimension1" content="Logged Out">
+
+
+
+        <meta name="hostname" content="github.com">
+    <meta name="user-login" content="">
+
+        <meta name="expected-hostname" content="github.com">
+      <meta name="js-proxy-site-detection-payload" content="N2EwMDI4NDlkOGY4ZWY5OWJjYWQ1NDQ1NjRhOWMxNDg0NzNmN2IwYmRjOWIxMWUwOTY4NmJjZGRjMGI1YjE1NXx7InJlbW90ZV9hZGRyZXNzIjoiMjE3LjYuMjEyLjEzOCIsInJlcXVlc3RfaWQiOiJEOTA2RDQ4QTo2Qjg5OjIyNUU1ODY6NTZEODJGOEMifQ==">
+
+      <link rel="mask-icon" href="https://assets-cdn.github.com/pinned-octocat.svg" color="#4078c0">
+      <link rel="icon" type="image/x-icon" href="https://assets-cdn.github.com/favicon.ico">
+
+    <meta content="80c998d8816164a29e0791941c3aa941d27d5c54" name="form-nonce" />
+
+    <meta http-equiv="x-pjax-version" content="b5f13da19cd72ae4f78b723c93c03e6b">
+    
+
+      
+  <meta name="description" content="lucene-solr - Mirror of Apache Lucene + Solr">
+  <meta name="go-import" content="github.com/apache/lucene-solr git https://github.com/apache/lucene-solr.git">
+
+  <meta content="47359" name="octolytics-dimension-user_id" /><meta content="apache" name="octolytics-dimension-user_login" /><meta content="50229487" name="octolytics-dimension-repository_id" /><meta content="apache/lucene-solr" name="octolytics-dimension-repository_nwo" /><meta content="true" name="octolytics-dimension-repository_public" /><meta content="false" name="octolytics-dimension-repository_is_fork" /><meta content="50229487" name="octolytics-dimension-repository_network_root_id" /><meta content="apache/lucene-solr" name="octolytics-dimension-repository_network_root_nwo" />
+  <link href="https://github.com/apache/lucene-solr/commits/master.atom" rel="alternate" title="Recent Commits to lucene-solr:master" type="application/atom+xml">
+
+
+      <link rel="canonical" href="https://github.com/apache/lucene-solr/blob/master/solr/example/files/conf/lang/stopwords_de.txt" data-pjax-transient>
+  </head>
+
+
+  <body class="logged_out   env-production  vis-public mirror  page-blob">
+    <a href="#start-of-content" tabindex="1" class="accessibility-aid js-skip-to-content">Skip to content</a>
+
+    
+    
+    
+
+
+
+      
+      <div class="header header-logged-out" role="banner">
+  <div class="container clearfix">
+
+    <a class="header-logo-wordmark" href="https://github.com/" data-ga-click="(Logged out) Header, go to homepage, icon:logo-wordmark">
+      <svg aria-hidden="true" class="octicon octicon-logo-github" height="28" role="img" version="1.1" viewBox="0 0 45 16" width="78"><path d="M8.64 5.19H4.88c-0.11 0-0.19 0.08-0.19 0.17v1.84c0 0.09 0.08 0.17 0.19 0.17h1.47v2.3s-0.33 0.11-1.25 0.11c-1.08 0-2.58-0.39-2.58-3.7s1.58-3.73 3.05-3.73c1.27 0 1.81 0.22 2.17 0.33 0.11 0.03 0.2-0.08 0.2-0.17l0.42-1.78c0-0.05-0.02-0.09-0.06-0.14-0.14-0.09-1.02-0.58-3.2-0.58C2.58 0 0 1.06 0 6.2s2.95 5.92 5.44 5.92c2.06 0 3.31-0.89 3.31-0.89 0.05-0.02 0.06-0.09 0.06-0.13V5.36c0-0.09-0.08-0.17-0.19-0.17h0.02zM27.7 0.44h-2.13c-0.09 0-0.17 0.08-0.17 0.17v4.09h-3.31V0.61c0-0.09-0.08-0.17-0.17-0.17h-2.13c-0.09 0-0.17 0.08-0.17 0.17v11.11c0 0.09 0.09 0.17 0.17 0.17h2.13c0.09 0 0.17-0.08 0.17-0.17V6.97h3.31l-0.02 4.75c0 0.09 0.08 0.17 0.17 0.17h2.13c0.09 0 0.17-0.08 0.17-0.17V0.61c0-0.09-0.08-0.17-0.17-0.17h0.02zM11.19 0.69c-0.77 0-1.38 0.61-1.38 1.38s0.61 1.38 1.38 1.38c0.75 0 1.36-0.61 1.36-1.38s-0.61-1.38-1.36-1.38z m1.22 3.55c0-0.09-0.08-0.17-0.17-0.17H10.11c-0.09 0-0.17 0.09-0.17 0.2 0 0 0 6.17 0 7.34 0 0.2 0.13 0.27 0.3 0.27 0 0 0.91 0 1.92 0 0.2 0 0.25-0.09 0.25-0.27 0-0.39 0-7.36 0-7.36v-0.02z m23.52-0.16h-2.09c-0.11 0-0.17 0.08-0.17 0.19v5.44s-0.55 0.39-1.3 0.39-0.97-0.34-0.97-1.09c0-0.73 0-4.75 0-4.75 0-0.09-0.08-0.17-0.17-0.17h-2.14c-0.09 0-0.17 0.08-0.17 0.17 0 0 0 2.91 0 5.11s1.23 2.75 2.92 2.75c1.39 0 2.52-0.77 2.52-0.77s0.05 0.39 0.08 0.45c0.02 0.05 0.09 0.09 0.16 0.09h1.34c0.11 0 0.17-0.08 0.17-0.17l0.02-7.47c0-0.09-0.08-0.17-0.19-0.17z m5.77-0.25c-1.2 0-2.02 0.53-2.02 0.53V0.59c0-0.09-0.08-0.17-0.17-0.17h-2.13c-0.09 0-0.17 0.08-0.17 0.17l-0.02 11.11c0 0.09 0.09 0.17 0.19 0.17h1.48c0.06 0 0.11-0.02 0.14-0.08 0.05-0.06 0.09-0.52 0.09-0.52s0.88 0.83 2.52 0.83c1.94 0 3.05-0.98 3.05-4.41s-1.77-3.88-2.97-3.88z m-0.83 6.27c-0.73-0.02-1.22-0.36-1.22-0.36V6.22s0.48-0.3 1.08-0.34c0.77-0.08 1.5 0.16 1.5 1.97 0 1.91-0.33 2.28-1.36 2.25z m-22.33-0.05c-0.09 0-0.33 0.05-0.58 0.05-0.78 0-1.05-0.36-1.05-0.83s0-3.13 0-3.13h1.59c0.09 0 0.16-0.08 0.16-0.19V4.25c0-0.09-0.08-0.17-0.16-0.17h-1.59V1.97c0-0.08-0.05-0.13-0.14-0.13H14.61c-0.09 0-0.14 0.05-0.14 0.13v2.17s-1.09 0.27-1.16 0.28c-0.08 0.02-0.13 0.09-0.13 0.17v1.36c0 0.11 0.08 0.19 0.17 0.19h1.11s0 1.44 0 3.28c0 2.44 1.7 2.69 2.86 2.69 0.53 0 1.17-0.17 1.27-0.22 0.06-0.02 0.09-0.09 0.09-0.16v-1.5c0-0.11-0.08-0.19-0.17-0.19h0.02z"></path></svg>
+    </a>
+
+    <div class="header-actions" role="navigation">
+        <a class="btn btn-primary" href="/join?source=header-repo" data-ga-click="(Logged out) Header, clicked Sign up, text:sign-up">Sign up</a>
+      <a class="btn" href="/login?return_to=%2Fapache%2Flucene-solr%2Fblob%2Fmaster%2Fsolr%2Fexample%2Ffiles%2Fconf%2Flang%2Fstopwords_de.txt" data-ga-click="(Logged out) Header, clicked Sign in, text:sign-in">Sign in</a>
+    </div>
+
+    <div class="site-search repo-scope js-site-search" role="search">
+      <!-- </textarea> --><!-- '"` --><form accept-charset="UTF-8" action="/apache/lucene-solr/search" class="js-site-search-form" data-global-search-url="/search" data-repo-search-url="/apache/lucene-solr/search" method="get"><div style="margin:0;padding:0;display:inline"><input name="utf8" type="hidden" value="&#x2713;" /></div>
+  <label class="js-chromeless-input-container form-control">
+    <div class="scope-badge">This repository</div>
+    <input type="text"
+      class="js-site-search-focus js-site-search-field is-clearable chromeless-input"
+      data-hotkey="s"
+      name="q"
+      placeholder="Search"
+      aria-label="Search this repository"
+      data-global-scope-placeholder="Search GitHub"
+      data-repo-scope-placeholder="Search"
+      tabindex="1"
+      autocapitalize="off">
+  </label>
+</form>
+    </div>
+
+      <ul class="header-nav left" role="navigation">
+          <li class="header-nav-item">
+            <a class="header-nav-link" href="/explore" data-ga-click="(Logged out) Header, go to explore, text:explore">Explore</a>
+          </li>
+          <li class="header-nav-item">
+            <a class="header-nav-link" href="/features" data-ga-click="(Logged out) Header, go to features, text:features">Features</a>
+          </li>
+          <li class="header-nav-item">
+            <a class="header-nav-link" href="https://enterprise.github.com/" data-ga-click="(Logged out) Header, go to enterprise, text:enterprise">Enterprise</a>
+          </li>
+          <li class="header-nav-item">
+            <a class="header-nav-link" href="/pricing" data-ga-click="(Logged out) Header, go to pricing, text:pricing">Pricing</a>
+          </li>
+      </ul>
+
+  </div>
+</div>
+
+
+
+    <div id="start-of-content" class="accessibility-aid"></div>
+
+      <div id="js-flash-container">
+</div>
+
+
+    <div role="main" class="main-content">
+        <div itemscope itemtype="http://schema.org/SoftwareSourceCode">
+    <div id="js-repo-pjax-container" class="context-loader-container js-repo-nav-next" data-pjax-container>
+      
+<div class="pagehead repohead instapaper_ignore readability-menu experiment-repo-nav">
+  <div class="container repohead-details-container">
+
+    
+
+<ul class="pagehead-actions">
+
+  <li>
+      <a href="/login?return_to=%2Fapache%2Flucene-solr"
+    class="btn btn-sm btn-with-count tooltipped tooltipped-n"
+    aria-label="You must be signed in to watch a repository" rel="nofollow">
+    <svg aria-hidden="true" class="octicon octicon-eye" height="16" role="img" version="1.1" viewBox="0 0 16 16" width="16"><path d="M8.06 2C3 2 0 8 0 8s3 6 8.06 6c4.94 0 7.94-6 7.94-6S13 2 8.06 2z m-0.06 10c-2.2 0-4-1.78-4-4 0-2.2 1.8-4 4-4 2.22 0 4 1.8 4 4 0 2.22-1.78 4-4 4z m2-4c0 1.11-0.89 2-2 2s-2-0.89-2-2 0.89-2 2-2 2 0.89 2 2z"></path></svg>
+    Watch
+  </a>
+  <a class="social-count" href="/apache/lucene-solr/watchers">
+    21
+  </a>
+
+  </li>
+
+  <li>
+      <a href="/login?return_to=%2Fapache%2Flucene-solr"
+    class="btn btn-sm btn-with-count tooltipped tooltipped-n"
+    aria-label="You must be signed in to star a repository" rel="nofollow">
+    <svg aria-hidden="true" class="octicon octicon-star" height="16" role="img" version="1.1" viewBox="0 0 14 16" width="14"><path d="M14 6l-4.9-0.64L7 1 4.9 5.36 0 6l3.6 3.26L2.67 14l4.33-2.33 4.33 2.33L10.4 9.26 14 6z"></path></svg>
+    Star
+  </a>
+
+    <a class="social-count js-social-count" href="/apache/lucene-solr/stargazers">
+      82
+    </a>
+
+  </li>
+
+  <li>
+      <a href="/login?return_to=%2Fapache%2Flucene-solr"
+        class="btn btn-sm btn-with-count tooltipped tooltipped-n"
+        aria-label="You must be signed in to fork a repository" rel="nofollow">
+        <svg aria-hidden="true" class="octicon octicon-repo-forked" height="16" role="img" version="1.1" viewBox="0 0 10 16" width="10"><path d="M8 1c-1.11 0-2 0.89-2 2 0 0.73 0.41 1.38 1 1.72v1.28L5 8 3 6v-1.28c0.59-0.34 1-0.98 1-1.72 0-1.11-0.89-2-2-2S0 1.89 0 3c0 0.73 0.41 1.38 1 1.72v1.78l3 3v1.78c-0.59 0.34-1 0.98-1 1.72 0 1.11 0.89 2 2 2s2-0.89 2-2c0-0.73-0.41-1.38-1-1.72V9.5l3-3V4.72c0.59-0.34 1-0.98 1-1.72 0-1.11-0.89-2-2-2zM2 4.2c-0.66 0-1.2-0.55-1.2-1.2s0.55-1.2 1.2-1.2 1.2 0.55 1.2 1.2-0.55 1.2-1.2 1.2z m3 10c-0.66 0-1.2-0.55-1.2-1.2s0.55-1.2 1.2-1.2 1.2 0.55 1.2 1.2-0.55 1.2-1.2 1.2z m3-10c-0.66 0-1.2-0.55-1.2-1.2s0.55-1.2 1.2-1.2 1.2 0.55 1.2 1.2-0.55 1.2-1.2 1.2z"></path></svg>
+        Fork
+      </a>
+
+    <a href="/apache/lucene-solr/network" class="social-count">
+      83
+    </a>
+  </li>
+</ul>
+
+    <h1 class="entry-title public repo-mirror">
+  <svg aria-hidden="true" class="octicon octicon-mirror" height="16" role="img" version="1.1" viewBox="0 0 16 16" width="16"><path d="M15.5 4.7L8.5 0 1.5 4.7c-0.3 0.19-0.5 0.45-0.5 0.8v10.5l7.5-4 7.5 4V5.5c0-0.34-0.2-0.61-0.5-0.8z m-0.5 9.8L9 11.25v-1.25h-1v1.25L2 14.5V5.5L8 1.5v4.5h1V1.5l6 4v9zM6 7h5V5l3 3-3 3V9H6v2L3 8l3-3v2z"></path></svg>
+  <span class="author" itemprop="author"><a href="/apache" class="url fn" rel="author">apache</a></span><!--
+--><span class="path-divider">/</span><!--
+--><strong itemprop="name"><a href="/apache/lucene-solr" data-pjax="#js-repo-pjax-container">lucene-solr</a></strong>
+
+  <span class="page-context-loader">
+    <img alt="" height="16" src="https://assets-cdn.github.com/images/spinners/octocat-spinner-32.gif" width="16" />
+  </span>
+
+    <span class="mirror-flag">
+      <span class="text">mirrored from <a href="git://git.apache.org/lucene-solr.git">git://git.apache.org/lucene-solr.git</a></span>
+    </span>
+</h1>
+
+  </div>
+  <div class="container">
+    
+<nav class="reponav js-repo-nav js-sidenav-container-pjax js-octicon-loaders"
+     itemscope
+     itemtype="http://schema.org/BreadcrumbList"
+     role="navigation"
+     data-pjax="#js-repo-pjax-container">
+
+  <span itemscope itemtype="http://schema.org/ListItem" itemprop="itemListElement">
+    <a href="/apache/lucene-solr" aria-selected="true" class="js-selected-navigation-item selected reponav-item" data-hotkey="g c" data-selected-links="repo_source repo_downloads repo_commits repo_releases repo_tags repo_branches /apache/lucene-solr" itemprop="url">
+      <svg aria-hidden="true" class="octicon octicon-code" height="16" role="img" version="1.1" viewBox="0 0 14 16" width="14"><path d="M9.5 3l-1.5 1.5 3.5 3.5L8 11.5l1.5 1.5 4.5-5L9.5 3zM4.5 3L0 8l4.5 5 1.5-1.5L2.5 8l3.5-3.5L4.5 3z"></path></svg>
+      <span itemprop="name">Code</span>
+      <meta itemprop="position" content="1">
+</a>  </span>
+
+
+  <span itemscope itemtype="http://schema.org/ListItem" itemprop="itemListElement">
+    <a href="/apache/lucene-solr/pulls" class="js-selected-navigation-item reponav-item" data-hotkey="g p" data-selected-links="repo_pulls /apache/lucene-solr/pulls" itemprop="url">
+      <svg aria-hidden="true" class="octicon octicon-git-pull-request" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M11 11.28c0-1.73 0-6.28 0-6.28-0.03-0.78-0.34-1.47-0.94-2.06s-1.28-0.91-2.06-0.94c0 0-1.02 0-1 0V0L4 3l3 3V4h1c0.27 0.02 0.48 0.11 0.69 0.31s0.3 0.42 0.31 0.69v6.28c-0.59 0.34-1 0.98-1 1.72 0 1.11 0.89 2 2 2s2-0.89 2-2c0-0.73-0.41-1.38-1-1.72z m-1 2.92c-0.66 0-1.2-0.55-1.2-1.2s0.55-1.2 1.2-1.2 1.2 0.55 1.2 1.2-0.55 1.2-1.2 1.2zM4 3c0-1.11-0.89-2-2-2S0 1.89 0 3c0 0.73 0.41 1.38 1 1.72 0 1.55 0 5.56 0 6.56-0.59 0.34-1 0.98-1 1.72 0 1.11 0.89 2 2 2s2-0.89 2-2c0-0.73-0.41-1.38-1-1.72V4.72c0.59-0.34 1-0.98 1-1.72z m-0.8 10c0 0.66-0.55 1.2-1.2 1.2s-1.2-0.55-1.2-1.2 0.55-1.2 1.2-1.2 1.2 0.55 1.2 1.2z m-1.2-8.8c-0.66 0-1.2-0.55-1.2-1.2s0.55-1.2 1.2-1.2 1.2 0.55 1.2 1.2-0.55 1.2-1.2 1.2z"></path></svg>
+      <span itemprop="name">Pull requests</span>
+      <span class="counter">12</span>
+      <meta itemprop="position" content="3">
+</a>  </span>
+
+
+  <a href="/apache/lucene-solr/pulse" class="js-selected-navigation-item reponav-item" data-selected-links="pulse /apache/lucene-solr/pulse">
+    <svg aria-hidden="true" class="octicon octicon-pulse" height="16" role="img" version="1.1" viewBox="0 0 14 16" width="14"><path d="M11.5 8L8.8 5.4 6.6 8.5 5.5 1.6 2.38 8H0V10h3.6L4.5 8.2l0.9 5.4L9 8.5l1.6 1.5H14V8H11.5z"></path></svg>
+    Pulse
+</a>
+  <a href="/apache/lucene-solr/graphs" class="js-selected-navigation-item reponav-item" data-selected-links="repo_graphs repo_contributors /apache/lucene-solr/graphs">
+    <svg aria-hidden="true" class="octicon octicon-graph" height="16" role="img" version="1.1" viewBox="0 0 16 16" width="16"><path d="M16 14v1H0V0h1v14h15z m-11-1H3V8h2v5z m4 0H7V3h2v10z m4 0H11V6h2v7z"></path></svg>
+    Graphs
+</a>
+
+</nav>
+
+  </div>
+</div>
+
+<div class="container new-discussion-timeline experiment-repo-nav">
+  <div class="repository-content">
+
+    
+
+<a href="/apache/lucene-solr/blob/6c06d4bc007454df09b2ace36ea839464b435be8/solr/example/files/conf/lang/stopwords_de.txt" class="hidden js-permalink-shortcut" data-hotkey="y">Permalink</a>
+
+<!-- blob contrib key: blob_contributors:v21:0bc50bfe3f2deebcc6f2e691f4b94310 -->
+
+<div class="file-navigation js-zeroclipboard-container">
+  
+<div class="select-menu js-menu-container js-select-menu left">
+  <button class="btn btn-sm select-menu-button js-menu-target css-truncate" data-hotkey="w"
+    title="master"
+    type="button" aria-label="Switch branches or tags" tabindex="0" aria-haspopup="true">
+    <i>Branch:</i>
+    <span class="js-select-button css-truncate-target">master</span>
+  </button>
+
+  <div class="select-menu-modal-holder js-menu-content js-navigation-container" data-pjax aria-hidden="true">
+
+    <div class="select-menu-modal">
+      <div class="select-menu-header">
+        <svg aria-label="Close" class="octicon octicon-x js-menu-close" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M7.48 8l3.75 3.75-1.48 1.48-3.75-3.75-3.75 3.75-1.48-1.48 3.75-3.75L0.77 4.25l1.48-1.48 3.75 3.75 3.75-3.75 1.48 1.48-3.75 3.75z"></path></svg>
+        <span class="select-menu-title">Switch branches/tags</span>
+      </div>
+
+      <div class="select-menu-filters">
+        <div class="select-menu-text-filter">
+          <input type="text" aria-label="Filter branches/tags" id="context-commitish-filter-field" class="js-filterable-field js-navigation-enable" placeholder="Filter branches/tags">
+        </div>
+        <div class="select-menu-tabs">
+          <ul>
+            <li class="select-menu-tab">
+              <a href="#" data-tab-filter="branches" data-filter-placeholder="Filter branches/tags" class="js-select-menu-tab" role="tab">Branches</a>
+            </li>
+            <li class="select-menu-tab">
+              <a href="#" data-tab-filter="tags" data-filter-placeholder="Find a tag…" class="js-select-menu-tab" role="tab">Tags</a>
+            </li>
+          </ul>
+        </div>
+      </div>
+
+      <div class="select-menu-list select-menu-tab-bucket js-select-menu-tab-bucket" data-tab-filter="branches" role="menu">
+
+        <div data-filterable-for="context-commitish-filter-field" data-filterable-type="substring">
+
+
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/apache/lucene-solr/blob/apiv2/solr/example/files/conf/lang/stopwords_de.txt"
+               data-name="apiv2"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text" title="apiv2">
+                apiv2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/apache/lucene-solr/blob/branch_3x/solr/example/files/conf/lang/stopwords_de.txt"
+               data-name="branch_3x"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text" title="branch_3x">
+                branch_3x
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/apache/lucene-solr/blob/branch_4x/solr/example/files/conf/lang/stopwords_de.txt"
+               data-name="branch_4x"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text" title="branch_4x">
+                branch_4x
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/apache/lucene-solr/blob/branch_5x/solr/example/files/conf/lang/stopwords_de.txt"
+               data-name="branch_5x"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text" title="branch_5x">
+                branch_5x
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/apache/lucene-solr/blob/branch_5_4/solr/example/files/conf/lang/stopwords_de.txt"
+               data-name="branch_5_4"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text" title="branch_5_4">
+                branch_5_4
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/apache/lucene-solr/blob/branch_5_5/solr/example/files/conf/lang/stopwords_de.txt"
+               data-name="branch_5_5"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text" title="branch_5_5">
+                branch_5_5
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/apache/lucene-solr/blob/branch_6x/solr/example/files/conf/lang/stopwords_de.txt"
+               data-name="branch_6x"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text" title="branch_6x">
+                branch_6x
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/apache/lucene-solr/blob/branch_6_0/solr/example/files/conf/lang/stopwords_de.txt"
+               data-name="branch_6_0"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text" title="branch_6_0">
+                branch_6_0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/apache/lucene-solr/blob/jira/SOLR-445/solr/example/files/conf/lang/stopwords_de.txt"
+               data-name="jira/SOLR-445"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text" title="jira/SOLR-445">
+                jira/SOLR-445
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/apache/lucene-solr/blob/jira/lucene-5438-nrt-replication/solr/example/files/conf/lang/stopwords_de.txt"
+               data-name="jira/lucene-5438-nrt-replication"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text" title="jira/lucene-5438-nrt-replication">
+                jira/lucene-5438-nrt-replication
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/apache/lucene-solr/blob/lucene-6835/solr/example/files/conf/lang/stopwords_de.txt"
+               data-name="lucene-6835"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text" title="lucene-6835">
+                lucene-6835
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/apache/lucene-solr/blob/lucene-6997/solr/example/files/conf/lang/stopwords_de.txt"
+               data-name="lucene-6997"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text" title="lucene-6997">
+                lucene-6997
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+               href="/apache/lucene-solr/blob/lucene-7015/solr/example/files/conf/lang/stopwords_de.txt"
+               data-name="lucene-7015"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text" title="lucene-7015">
+                lucene-7015
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open selected"
+               href="/apache/lucene-solr/blob/master/solr/example/files/conf/lang/stopwords_de.txt"
+               data-name="master"
+               data-skip-pjax="true"
+               rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target js-select-menu-filter-text" title="master">
+                master
+              </span>
+            </a>
+        </div>
+
+          <div class="select-menu-no-results">Nothing to show</div>
+      </div>
+
+      <div class="select-menu-list select-menu-tab-bucket js-select-menu-tab-bucket" data-tab-filter="tags">
+        <div data-filterable-for="context-commitish-filter-field" data-filterable-type="substring">
+
+
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/solr/1.4.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/solr/1.4.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/solr/1.4.0">
+                releases/solr/1.4.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/solr/1.3.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/solr/1.3.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/solr/1.3.0">
+                releases/solr/1.3.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/solr/1.2.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/solr/1.2.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/solr/1.2.0">
+                releases/solr/1.2.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/solr/1.1.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/solr/1.1.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/solr/1.1.0">
+                releases/solr/1.1.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/3.0.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/3.0.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/3.0.1">
+                releases/lucene/3.0.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/3.0.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/3.0.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/3.0.0">
+                releases/lucene/3.0.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/2.9.2/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/2.9.2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/2.9.2">
+                releases/lucene/2.9.2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/2.9.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/2.9.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/2.9.1">
+                releases/lucene/2.9.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/2.9.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/2.9.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/2.9.0">
+                releases/lucene/2.9.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/2.4.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/2.4.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/2.4.1">
+                releases/lucene/2.4.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/2.4.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/2.4.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/2.4.0">
+                releases/lucene/2.4.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/2.3.2/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/2.3.2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/2.3.2">
+                releases/lucene/2.3.2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/2.3.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/2.3.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/2.3.1">
+                releases/lucene/2.3.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/2.3.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/2.3.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/2.3.0">
+                releases/lucene/2.3.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/2.2.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/2.2.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/2.2.0">
+                releases/lucene/2.2.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/2.1.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/2.1.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/2.1.0">
+                releases/lucene/2.1.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/2.0.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/2.0.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/2.0.0">
+                releases/lucene/2.0.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.9.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.9.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.9.1">
+                releases/lucene/1.9.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.9/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.9"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.9">
+                releases/lucene/1.9
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.9-rc1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.9-rc1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.9-rc1">
+                releases/lucene/1.9-rc1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.4.3/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.4.3"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.4.3">
+                releases/lucene/1.4.3
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.4.2/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.4.2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.4.2">
+                releases/lucene/1.4.2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.4.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.4.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.4.1">
+                releases/lucene/1.4.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.4/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.4"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.4">
+                releases/lucene/1.4
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.4-rc3/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.4-rc3"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.4-rc3">
+                releases/lucene/1.4-rc3
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.4-rc2/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.4-rc2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.4-rc2">
+                releases/lucene/1.4-rc2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.4-rc1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.4-rc1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.4-rc1">
+                releases/lucene/1.4-rc1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.3/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.3"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.3">
+                releases/lucene/1.3
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.3-rc3/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.3-rc3"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.3-rc3">
+                releases/lucene/1.3-rc3
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.3-rc2/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.3-rc2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.3-rc2">
+                releases/lucene/1.3-rc2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.3-rc1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.3-rc1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.3-rc1">
+                releases/lucene/1.3-rc1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.2/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.2">
+                releases/lucene/1.2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.2-rc5/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.2-rc5"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.2-rc5">
+                releases/lucene/1.2-rc5
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.2-rc4/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.2-rc4"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.2-rc4">
+                releases/lucene/1.2-rc4
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.2-rc3/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.2-rc3"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.2-rc3">
+                releases/lucene/1.2-rc3
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.2-rc2/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.2-rc2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.2-rc2">
+                releases/lucene/1.2-rc2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.2-rc1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.2-rc1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.2-rc1">
+                releases/lucene/1.2-rc1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene/1.0.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene/1.0.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene/1.0.1">
+                releases/lucene/1.0.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/5.5.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/5.5.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/5.5.0">
+                releases/lucene-solr/5.5.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/5.4.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/5.4.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/5.4.1">
+                releases/lucene-solr/5.4.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/5.4.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/5.4.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/5.4.0">
+                releases/lucene-solr/5.4.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/5.3.2/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/5.3.2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/5.3.2">
+                releases/lucene-solr/5.3.2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/5.3.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/5.3.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/5.3.1">
+                releases/lucene-solr/5.3.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/5.3.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/5.3.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/5.3.0">
+                releases/lucene-solr/5.3.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/5.2.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/5.2.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/5.2.1">
+                releases/lucene-solr/5.2.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/5.2.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/5.2.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/5.2.0">
+                releases/lucene-solr/5.2.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/5.1.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/5.1.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/5.1.0">
+                releases/lucene-solr/5.1.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/5.0.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/5.0.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/5.0.0">
+                releases/lucene-solr/5.0.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.10.4/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.10.4"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.10.4">
+                releases/lucene-solr/4.10.4
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.10.3/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.10.3"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.10.3">
+                releases/lucene-solr/4.10.3
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.10.2/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.10.2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.10.2">
+                releases/lucene-solr/4.10.2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.10.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.10.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.10.1">
+                releases/lucene-solr/4.10.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.10.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.10.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.10.0">
+                releases/lucene-solr/4.10.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.9.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.9.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.9.1">
+                releases/lucene-solr/4.9.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.9.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.9.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.9.0">
+                releases/lucene-solr/4.9.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.8.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.8.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.8.1">
+                releases/lucene-solr/4.8.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.8.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.8.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.8.0">
+                releases/lucene-solr/4.8.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.7.2/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.7.2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.7.2">
+                releases/lucene-solr/4.7.2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.7.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.7.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.7.1">
+                releases/lucene-solr/4.7.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.7.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.7.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.7.0">
+                releases/lucene-solr/4.7.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.6.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.6.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.6.1">
+                releases/lucene-solr/4.6.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.6.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.6.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.6.0">
+                releases/lucene-solr/4.6.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.5.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.5.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.5.1">
+                releases/lucene-solr/4.5.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.5.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.5.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.5.0">
+                releases/lucene-solr/4.5.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.4.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.4.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.4.0">
+                releases/lucene-solr/4.4.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.3.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.3.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.3.1">
+                releases/lucene-solr/4.3.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.3.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.3.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.3.0">
+                releases/lucene-solr/4.3.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.2.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.2.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.2.1">
+                releases/lucene-solr/4.2.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.2.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.2.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.2.0">
+                releases/lucene-solr/4.2.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.1.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.1.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.1.0">
+                releases/lucene-solr/4.1.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.0.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.0.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.0.0">
+                releases/lucene-solr/4.0.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.0.0-beta/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.0.0-beta"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.0.0-beta">
+                releases/lucene-solr/4.0.0-beta
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/4.0.0-alpha/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/4.0.0-alpha"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/4.0.0-alpha">
+                releases/lucene-solr/4.0.0-alpha
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/3.6.2/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/3.6.2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/3.6.2">
+                releases/lucene-solr/3.6.2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/3.6.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/3.6.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/3.6.1">
+                releases/lucene-solr/3.6.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/3.6.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/3.6.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/3.6.0">
+                releases/lucene-solr/3.6.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/3.5.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/3.5.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/3.5.0">
+                releases/lucene-solr/3.5.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/3.4.0/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/3.4.0"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/3.4.0">
+                releases/lucene-solr/3.4.0
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/3.3/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/3.3"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/3.3">
+                releases/lucene-solr/3.3
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/3.2/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/3.2"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/3.2">
+                releases/lucene-solr/3.2
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/releases/lucene-solr/3.1/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="releases/lucene-solr/3.1"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="releases/lucene-solr/3.1">
+                releases/lucene-solr/3.1
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/solr/cloud/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/solr/cloud"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/solr/cloud">
+                history/branches/solr/cloud
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/solr/branch-1.3/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/solr/branch-1.3"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/solr/branch-1.3">
+                history/branches/solr/branch-1.3
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/throwawaybranch/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/throwawaybranch"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/throwawaybranch">
+                history/branches/lucene-solr/throwawaybranch
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/solrcloud/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/solrcloud"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/solrcloud">
+                history/branches/lucene-solr/solrcloud
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/solr7790/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/solr7790"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/solr7790">
+                history/branches/lucene-solr/solr7790
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/solr7787/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/solr7787"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/solr7787">
+                history/branches/lucene-solr/solr7787
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/solr5914/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/solr5914"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/solr5914">
+                history/branches/lucene-solr/solr5914
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/solr5473/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/solr5473"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/solr5473">
+                history/branches/lucene-solr/solr5473
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/solr4470/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/solr4470"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/solr4470">
+                history/branches/lucene-solr/solr4470
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/solr3733/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/solr3733"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/solr3733">
+                history/branches/lucene-solr/solr3733
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/solr_3159_jetty8/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/solr_3159_jetty8"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/solr_3159_jetty8">
+                history/branches/lucene-solr/solr_3159_jetty8
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/solr2452/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/solr2452"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/solr2452">
+                history/branches/lucene-solr/solr2452
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/solr2193/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/solr2193"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/solr2193">
+                history/branches/lucene-solr/solr2193
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/solr_guice_restlet/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/solr_guice_restlet"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/solr_guice_restlet">
+                history/branches/lucene-solr/solr_guice_restlet
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/solr-5473/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/solr-5473"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/solr-5473">
+                history/branches/lucene-solr/solr-5473
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/slowclosing/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/slowclosing"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/slowclosing">
+                history/branches/lucene-solr/slowclosing
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/security/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/security"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/security">
+                history/branches/lucene-solr/security
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/realtime_search/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/realtime_search"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/realtime_search">
+                history/branches/lucene-solr/realtime_search
+              </span>
+            </a>
+            <a class="select-menu-item js-navigation-item js-navigation-open "
+              href="/apache/lucene-solr/tree/history/branches/lucene-solr/pseudo/solr/example/files/conf/lang/stopwords_de.txt"
+              data-name="history/branches/lucene-solr/pseudo"
+              data-skip-pjax="true"
+              rel="nofollow">
+              <svg aria-hidden="true" class="octicon octicon-check select-menu-item-icon" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M12 5L4 13 0 9l1.5-1.5 2.5 2.5 6.5-6.5 1.5 1.5z"></path></svg>
+              <span class="select-menu-item-text css-truncate-target" title="history/branches/lucene-solr/pseudo">
+                history/branches/lucene-solr/pseudo
+              </span>
+            </a>
+        </div>
+
+        <div class="select-menu-no-results">Nothing to show</div>
+      </div>
+
+    </div>
+  </div>
+</div>
+
+  <div class="btn-group right">
+    <a href="/apache/lucene-solr/find/master"
+          class="js-show-file-finder btn btn-sm"
+          data-pjax
+          data-hotkey="t">
+      Find file
+    </a>
+    <button aria-label="Copy file path to clipboard" class="js-zeroclipboard btn btn-sm zeroclipboard-button tooltipped tooltipped-s" data-copied-hint="Copied!" type="button">Copy path</button>
+  </div>
+  <div class="breadcrumb js-zeroclipboard-target">
+    <span class="repo-root js-repo-root"><span class="js-path-segment"><a href="/apache/lucene-solr"><span>lucene-solr</span></a></span></span><span class="separator">/</span><span class="js-path-segment"><a href="/apache/lucene-solr/tree/master/solr"><span>solr</span></a></span><span class="separator">/</span><span class="js-path-segment"><a href="/apache/lucene-solr/tree/master/solr/example"><span>example</span></a></span><span class="separator">/</span><span class="js-path-segment"><a href="/apache/lucene-solr/tree/master/solr/example/files"><span>files</span></a></span><span class="separator">/</span><span class="js-path-segment"><a href="/apache/lucene-solr/tree/master/solr/example/files/conf"><span>conf</span></a></span><span class="separator">/</span><span class="js-path-segment"><a href="/apache/lucene-solr/tree/master/solr/example/files/conf/lang"><span>lang</span></a></span><span class="separator">/</span><strong class="final-path">stopwords_de.txt</strong>
+  </div>
+</div>
+
+
+  <div class="commit-tease">
+      <span class="right">
+        <a class="commit-tease-sha" href="/apache/lucene-solr/commit/be4680abb8ca60d4206c17b9b9cb256a4406a633" data-pjax>
+          be4680a
+        </a>
+        <time datetime="2015-05-22T01:23:27Z" is="relative-time">May 22, 2015</time>
+      </span>
+      <div>
+        <img alt="@erikhatcher" class="avatar" height="20" src="https://avatars1.githubusercontent.com/u/209189?v=3&amp;s=40" width="20" />
+        <a href="/erikhatcher" class="user-mention" rel="contributor">erikhatcher</a>
+          <a href="/apache/lucene-solr/commit/be4680abb8ca60d4206c17b9b9cb256a4406a633" class="message" data-pjax="true" title="SOLR-7465: New file indexing example, under example/files
+
+git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1680973 13f79535-47bb-0310-9956-ffa450edef68">SOLR-7465: New file indexing example, under example/files</a>
+      </div>
+
+    <div class="commit-tease-contributors">
+      <button type="button" class="btn-link muted-link contributors-toggle" data-facebox="#blob_contributors_box">
+        <strong>1</strong>
+         contributor
+      </button>
+      
+    </div>
+
+    <div id="blob_contributors_box" style="display:none">
+      <h2 class="facebox-header" data-facebox-id="facebox-header">Users who have contributed to this file</h2>
+      <ul class="facebox-user-list" data-facebox-id="facebox-description">
+          <li class="facebox-user-list-item">
+            <img alt="@erikhatcher" height="24" src="https://avatars3.githubusercontent.com/u/209189?v=3&amp;s=48" width="24" />
+            <a href="/erikhatcher">erikhatcher</a>
+          </li>
+      </ul>
+    </div>
+  </div>
+
+<div class="file">
+  <div class="file-header">
+  <div class="file-actions">
+
+    <div class="btn-group">
+      <a href="/apache/lucene-solr/raw/master/solr/example/files/conf/lang/stopwords_de.txt" class="btn btn-sm " id="raw-url">Raw</a>
+        <a href="/apache/lucene-solr/blame/master/solr/example/files/conf/lang/stopwords_de.txt" class="btn btn-sm js-update-url-with-hash">Blame</a>
+      <a href="/apache/lucene-solr/commits/master/solr/example/files/conf/lang/stopwords_de.txt" class="btn btn-sm " rel="nofollow">History</a>
+    </div>
+
+
+        <button type="button" class="btn-octicon disabled tooltipped tooltipped-nw"
+          aria-label="You must be signed in to make or propose changes">
+          <svg aria-hidden="true" class="octicon octicon-pencil" height="16" role="img" version="1.1" viewBox="0 0 14 16" width="14"><path d="M0 12v3h3l8-8-3-3L0 12z m3 2H1V12h1v1h1v1z m10.3-9.3l-1.3 1.3-3-3 1.3-1.3c0.39-0.39 1.02-0.39 1.41 0l1.59 1.59c0.39 0.39 0.39 1.02 0 1.41z"></path></svg>
+        </button>
+        <button type="button" class="btn-octicon btn-octicon-danger disabled tooltipped tooltipped-nw"
+          aria-label="You must be signed in to make or propose changes">
+          <svg aria-hidden="true" class="octicon octicon-trashcan" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M10 2H8c0-0.55-0.45-1-1-1H4c-0.55 0-1 0.45-1 1H1c-0.55 0-1 0.45-1 1v1c0 0.55 0.45 1 1 1v9c0 0.55 0.45 1 1 1h7c0.55 0 1-0.45 1-1V5c0.55 0 1-0.45 1-1v-1c0-0.55-0.45-1-1-1z m-1 12H2V5h1v8h1V5h1v8h1V5h1v8h1V5h1v9z m1-10H1v-1h9v1z"></path></svg>
+        </button>
+  </div>
+
+  <div class="file-info">
+      295 lines (243 sloc)
+      <span class="file-info-divider"></span>
+    4.47 KB
+  </div>
+</div>
+
+  
+
+  <div itemprop="text" class="blob-wrapper data type-text">
+      <table class="highlight tab-size js-file-line-container" data-tab-size="8">
+      <tr>
+        <td id="L1" class="blob-num js-line-number" data-line-number="1"></td>
+        <td id="LC1" class="blob-code blob-code-inner js-file-line"> | From svn.tartarus.org/snowball/trunk/website/algorithms/german/stop.txt</td>
+      </tr>
+      <tr>
+        <td id="L2" class="blob-num js-line-number" data-line-number="2"></td>
+        <td id="LC2" class="blob-code blob-code-inner js-file-line"> | This file is distributed under the BSD License.</td>
+      </tr>
+      <tr>
+        <td id="L3" class="blob-num js-line-number" data-line-number="3"></td>
+        <td id="LC3" class="blob-code blob-code-inner js-file-line"> | See http://snowball.tartarus.org/license.php</td>
+      </tr>
+      <tr>
+        <td id="L4" class="blob-num js-line-number" data-line-number="4"></td>
+        <td id="LC4" class="blob-code blob-code-inner js-file-line"> | Also see http://www.opensource.org/licenses/bsd-license.html</td>
+      </tr>
+      <tr>
+        <td id="L5" class="blob-num js-line-number" data-line-number="5"></td>
+        <td id="LC5" class="blob-code blob-code-inner js-file-line"> |  - Encoding was converted to UTF-8.</td>
+      </tr>
+      <tr>
+        <td id="L6" class="blob-num js-line-number" data-line-number="6"></td>
+        <td id="LC6" class="blob-code blob-code-inner js-file-line"> |  - This notice was added.</td>
+      </tr>
+      <tr>
+        <td id="L7" class="blob-num js-line-number" data-line-number="7"></td>
+        <td id="LC7" class="blob-code blob-code-inner js-file-line"> |</td>
+      </tr>
+      <tr>
+        <td id="L8" class="blob-num js-line-number" data-line-number="8"></td>
+        <td id="LC8" class="blob-code blob-code-inner js-file-line"> | NOTE: To use this file with StopFilterFactory, you must specify format=&quot;snowball&quot;</td>
+      </tr>
+      <tr>
+        <td id="L9" class="blob-num js-line-number" data-line-number="9"></td>
+        <td id="LC9" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L10" class="blob-num js-line-number" data-line-number="10"></td>
+        <td id="LC10" class="blob-code blob-code-inner js-file-line"> | A German stop word list. Comments begin with vertical bar. Each stop</td>
+      </tr>
+      <tr>
+        <td id="L11" class="blob-num js-line-number" data-line-number="11"></td>
+        <td id="LC11" class="blob-code blob-code-inner js-file-line"> | word is at the start of a line.</td>
+      </tr>
+      <tr>
+        <td id="L12" class="blob-num js-line-number" data-line-number="12"></td>
+        <td id="LC12" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L13" class="blob-num js-line-number" data-line-number="13"></td>
+        <td id="LC13" class="blob-code blob-code-inner js-file-line"> | The number of forms in this list is reduced significantly by passing it</td>
+      </tr>
+      <tr>
+        <td id="L14" class="blob-num js-line-number" data-line-number="14"></td>
+        <td id="LC14" class="blob-code blob-code-inner js-file-line"> | through the German stemmer.</td>
+      </tr>
+      <tr>
+        <td id="L15" class="blob-num js-line-number" data-line-number="15"></td>
+        <td id="LC15" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L16" class="blob-num js-line-number" data-line-number="16"></td>
+        <td id="LC16" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L17" class="blob-num js-line-number" data-line-number="17"></td>
+        <td id="LC17" class="blob-code blob-code-inner js-file-line">aber           |  but</td>
+      </tr>
+      <tr>
+        <td id="L18" class="blob-num js-line-number" data-line-number="18"></td>
+        <td id="LC18" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L19" class="blob-num js-line-number" data-line-number="19"></td>
+        <td id="LC19" class="blob-code blob-code-inner js-file-line">alle           |  all</td>
+      </tr>
+      <tr>
+        <td id="L20" class="blob-num js-line-number" data-line-number="20"></td>
+        <td id="LC20" class="blob-code blob-code-inner js-file-line">allem</td>
+      </tr>
+      <tr>
+        <td id="L21" class="blob-num js-line-number" data-line-number="21"></td>
+        <td id="LC21" class="blob-code blob-code-inner js-file-line">allen</td>
+      </tr>
+      <tr>
+        <td id="L22" class="blob-num js-line-number" data-line-number="22"></td>
+        <td id="LC22" class="blob-code blob-code-inner js-file-line">aller</td>
+      </tr>
+      <tr>
+        <td id="L23" class="blob-num js-line-number" data-line-number="23"></td>
+        <td id="LC23" class="blob-code blob-code-inner js-file-line">alles</td>
+      </tr>
+      <tr>
+        <td id="L24" class="blob-num js-line-number" data-line-number="24"></td>
+        <td id="LC24" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L25" class="blob-num js-line-number" data-line-number="25"></td>
+        <td id="LC25" class="blob-code blob-code-inner js-file-line">als            |  than, as</td>
+      </tr>
+      <tr>
+        <td id="L26" class="blob-num js-line-number" data-line-number="26"></td>
+        <td id="LC26" class="blob-code blob-code-inner js-file-line">also           |  so</td>
+      </tr>
+      <tr>
+        <td id="L27" class="blob-num js-line-number" data-line-number="27"></td>
+        <td id="LC27" class="blob-code blob-code-inner js-file-line">am             |  an + dem</td>
+      </tr>
+      <tr>
+        <td id="L28" class="blob-num js-line-number" data-line-number="28"></td>
+        <td id="LC28" class="blob-code blob-code-inner js-file-line">an             |  at</td>
+      </tr>
+      <tr>
+        <td id="L29" class="blob-num js-line-number" data-line-number="29"></td>
+        <td id="LC29" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L30" class="blob-num js-line-number" data-line-number="30"></td>
+        <td id="LC30" class="blob-code blob-code-inner js-file-line">ander          |  other</td>
+      </tr>
+      <tr>
+        <td id="L31" class="blob-num js-line-number" data-line-number="31"></td>
+        <td id="LC31" class="blob-code blob-code-inner js-file-line">andere</td>
+      </tr>
+      <tr>
+        <td id="L32" class="blob-num js-line-number" data-line-number="32"></td>
+        <td id="LC32" class="blob-code blob-code-inner js-file-line">anderem</td>
+      </tr>
+      <tr>
+        <td id="L33" class="blob-num js-line-number" data-line-number="33"></td>
+        <td id="LC33" class="blob-code blob-code-inner js-file-line">anderen</td>
+      </tr>
+      <tr>
+        <td id="L34" class="blob-num js-line-number" data-line-number="34"></td>
+        <td id="LC34" class="blob-code blob-code-inner js-file-line">anderer</td>
+      </tr>
+      <tr>
+        <td id="L35" class="blob-num js-line-number" data-line-number="35"></td>
+        <td id="LC35" class="blob-code blob-code-inner js-file-line">anderes</td>
+      </tr>
+      <tr>
+        <td id="L36" class="blob-num js-line-number" data-line-number="36"></td>
+        <td id="LC36" class="blob-code blob-code-inner js-file-line">anderm</td>
+      </tr>
+      <tr>
+        <td id="L37" class="blob-num js-line-number" data-line-number="37"></td>
+        <td id="LC37" class="blob-code blob-code-inner js-file-line">andern</td>
+      </tr>
+      <tr>
+        <td id="L38" class="blob-num js-line-number" data-line-number="38"></td>
+        <td id="LC38" class="blob-code blob-code-inner js-file-line">anderr</td>
+      </tr>
+      <tr>
+        <td id="L39" class="blob-num js-line-number" data-line-number="39"></td>
+        <td id="LC39" class="blob-code blob-code-inner js-file-line">anders</td>
+      </tr>
+      <tr>
+        <td id="L40" class="blob-num js-line-number" data-line-number="40"></td>
+        <td id="LC40" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L41" class="blob-num js-line-number" data-line-number="41"></td>
+        <td id="LC41" class="blob-code blob-code-inner js-file-line">auch           |  also</td>
+      </tr>
+      <tr>
+        <td id="L42" class="blob-num js-line-number" data-line-number="42"></td>
+        <td id="LC42" class="blob-code blob-code-inner js-file-line">auf            |  on</td>
+      </tr>
+      <tr>
+        <td id="L43" class="blob-num js-line-number" data-line-number="43"></td>
+        <td id="LC43" class="blob-code blob-code-inner js-file-line">aus            |  out of</td>
+      </tr>
+      <tr>
+        <td id="L44" class="blob-num js-line-number" data-line-number="44"></td>
+        <td id="LC44" class="blob-code blob-code-inner js-file-line">bei            |  by</td>
+      </tr>
+      <tr>
+        <td id="L45" class="blob-num js-line-number" data-line-number="45"></td>
+        <td id="LC45" class="blob-code blob-code-inner js-file-line">bin            |  am</td>
+      </tr>
+      <tr>
+        <td id="L46" class="blob-num js-line-number" data-line-number="46"></td>
+        <td id="LC46" class="blob-code blob-code-inner js-file-line">bis            |  until</td>
+      </tr>
+      <tr>
+        <td id="L47" class="blob-num js-line-number" data-line-number="47"></td>
+        <td id="LC47" class="blob-code blob-code-inner js-file-line">bist           |  art</td>
+      </tr>
+      <tr>
+        <td id="L48" class="blob-num js-line-number" data-line-number="48"></td>
+        <td id="LC48" class="blob-code blob-code-inner js-file-line">da             |  there</td>
+      </tr>
+      <tr>
+        <td id="L49" class="blob-num js-line-number" data-line-number="49"></td>
+        <td id="LC49" class="blob-code blob-code-inner js-file-line">damit          |  with it</td>
+      </tr>
+      <tr>
+        <td id="L50" class="blob-num js-line-number" data-line-number="50"></td>
+        <td id="LC50" class="blob-code blob-code-inner js-file-line">dann           |  then</td>
+      </tr>
+      <tr>
+        <td id="L51" class="blob-num js-line-number" data-line-number="51"></td>
+        <td id="LC51" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L52" class="blob-num js-line-number" data-line-number="52"></td>
+        <td id="LC52" class="blob-code blob-code-inner js-file-line">der            |  the</td>
+      </tr>
+      <tr>
+        <td id="L53" class="blob-num js-line-number" data-line-number="53"></td>
+        <td id="LC53" class="blob-code blob-code-inner js-file-line">den</td>
+      </tr>
+      <tr>
+        <td id="L54" class="blob-num js-line-number" data-line-number="54"></td>
+        <td id="LC54" class="blob-code blob-code-inner js-file-line">des</td>
+      </tr>
+      <tr>
+        <td id="L55" class="blob-num js-line-number" data-line-number="55"></td>
+        <td id="LC55" class="blob-code blob-code-inner js-file-line">dem</td>
+      </tr>
+      <tr>
+        <td id="L56" class="blob-num js-line-number" data-line-number="56"></td>
+        <td id="LC56" class="blob-code blob-code-inner js-file-line">die</td>
+      </tr>
+      <tr>
+        <td id="L57" class="blob-num js-line-number" data-line-number="57"></td>
+        <td id="LC57" class="blob-code blob-code-inner js-file-line">das</td>
+      </tr>
+      <tr>
+        <td id="L58" class="blob-num js-line-number" data-line-number="58"></td>
+        <td id="LC58" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L59" class="blob-num js-line-number" data-line-number="59"></td>
+        <td id="LC59" class="blob-code blob-code-inner js-file-line">daß            |  that</td>
+      </tr>
+      <tr>
+        <td id="L60" class="blob-num js-line-number" data-line-number="60"></td>
+        <td id="LC60" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L61" class="blob-num js-line-number" data-line-number="61"></td>
+        <td id="LC61" class="blob-code blob-code-inner js-file-line">derselbe       |  the same</td>
+      </tr>
+      <tr>
+        <td id="L62" class="blob-num js-line-number" data-line-number="62"></td>
+        <td id="LC62" class="blob-code blob-code-inner js-file-line">derselben</td>
+      </tr>
+      <tr>
+        <td id="L63" class="blob-num js-line-number" data-line-number="63"></td>
+        <td id="LC63" class="blob-code blob-code-inner js-file-line">denselben</td>
+      </tr>
+      <tr>
+        <td id="L64" class="blob-num js-line-number" data-line-number="64"></td>
+        <td id="LC64" class="blob-code blob-code-inner js-file-line">desselben</td>
+      </tr>
+      <tr>
+        <td id="L65" class="blob-num js-line-number" data-line-number="65"></td>
+        <td id="LC65" class="blob-code blob-code-inner js-file-line">demselben</td>
+      </tr>
+      <tr>
+        <td id="L66" class="blob-num js-line-number" data-line-number="66"></td>
+        <td id="LC66" class="blob-code blob-code-inner js-file-line">dieselbe</td>
+      </tr>
+      <tr>
+        <td id="L67" class="blob-num js-line-number" data-line-number="67"></td>
+        <td id="LC67" class="blob-code blob-code-inner js-file-line">dieselben</td>
+      </tr>
+      <tr>
+        <td id="L68" class="blob-num js-line-number" data-line-number="68"></td>
+        <td id="LC68" class="blob-code blob-code-inner js-file-line">dasselbe</td>
+      </tr>
+      <tr>
+        <td id="L69" class="blob-num js-line-number" data-line-number="69"></td>
+        <td id="LC69" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L70" class="blob-num js-line-number" data-line-number="70"></td>
+        <td id="LC70" class="blob-code blob-code-inner js-file-line">dazu           |  to that</td>
+      </tr>
+      <tr>
+        <td id="L71" class="blob-num js-line-number" data-line-number="71"></td>
+        <td id="LC71" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L72" class="blob-num js-line-number" data-line-number="72"></td>
+        <td id="LC72" class="blob-code blob-code-inner js-file-line">dein           |  thy</td>
+      </tr>
+      <tr>
+        <td id="L73" class="blob-num js-line-number" data-line-number="73"></td>
+        <td id="LC73" class="blob-code blob-code-inner js-file-line">deine</td>
+      </tr>
+      <tr>
+        <td id="L74" class="blob-num js-line-number" data-line-number="74"></td>
+        <td id="LC74" class="blob-code blob-code-inner js-file-line">deinem</td>
+      </tr>
+      <tr>
+        <td id="L75" class="blob-num js-line-number" data-line-number="75"></td>
+        <td id="LC75" class="blob-code blob-code-inner js-file-line">deinen</td>
+      </tr>
+      <tr>
+        <td id="L76" class="blob-num js-line-number" data-line-number="76"></td>
+        <td id="LC76" class="blob-code blob-code-inner js-file-line">deiner</td>
+      </tr>
+      <tr>
+        <td id="L77" class="blob-num js-line-number" data-line-number="77"></td>
+        <td id="LC77" class="blob-code blob-code-inner js-file-line">deines</td>
+      </tr>
+      <tr>
+        <td id="L78" class="blob-num js-line-number" data-line-number="78"></td>
+        <td id="LC78" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L79" class="blob-num js-line-number" data-line-number="79"></td>
+        <td id="LC79" class="blob-code blob-code-inner js-file-line">denn           |  because</td>
+      </tr>
+      <tr>
+        <td id="L80" class="blob-num js-line-number" data-line-number="80"></td>
+        <td id="LC80" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L81" class="blob-num js-line-number" data-line-number="81"></td>
+        <td id="LC81" class="blob-code blob-code-inner js-file-line">derer          |  of those</td>
+      </tr>
+      <tr>
+        <td id="L82" class="blob-num js-line-number" data-line-number="82"></td>
+        <td id="LC82" class="blob-code blob-code-inner js-file-line">dessen         |  of him</td>
+      </tr>
+      <tr>
+        <td id="L83" class="blob-num js-line-number" data-line-number="83"></td>
+        <td id="LC83" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L84" class="blob-num js-line-number" data-line-number="84"></td>
+        <td id="LC84" class="blob-code blob-code-inner js-file-line">dich           |  thee</td>
+      </tr>
+      <tr>
+        <td id="L85" class="blob-num js-line-number" data-line-number="85"></td>
+        <td id="LC85" class="blob-code blob-code-inner js-file-line">dir            |  to thee</td>
+      </tr>
+      <tr>
+        <td id="L86" class="blob-num js-line-number" data-line-number="86"></td>
+        <td id="LC86" class="blob-code blob-code-inner js-file-line">du             |  thou</td>
+      </tr>
+      <tr>
+        <td id="L87" class="blob-num js-line-number" data-line-number="87"></td>
+        <td id="LC87" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L88" class="blob-num js-line-number" data-line-number="88"></td>
+        <td id="LC88" class="blob-code blob-code-inner js-file-line">dies           |  this</td>
+      </tr>
+      <tr>
+        <td id="L89" class="blob-num js-line-number" data-line-number="89"></td>
+        <td id="LC89" class="blob-code blob-code-inner js-file-line">diese</td>
+      </tr>
+      <tr>
+        <td id="L90" class="blob-num js-line-number" data-line-number="90"></td>
+        <td id="LC90" class="blob-code blob-code-inner js-file-line">diesem</td>
+      </tr>
+      <tr>
+        <td id="L91" class="blob-num js-line-number" data-line-number="91"></td>
+        <td id="LC91" class="blob-code blob-code-inner js-file-line">diesen</td>
+      </tr>
+      <tr>
+        <td id="L92" class="blob-num js-line-number" data-line-number="92"></td>
+        <td id="LC92" class="blob-code blob-code-inner js-file-line">dieser</td>
+      </tr>
+      <tr>
+        <td id="L93" class="blob-num js-line-number" data-line-number="93"></td>
+        <td id="LC93" class="blob-code blob-code-inner js-file-line">dieses</td>
+      </tr>
+      <tr>
+        <td id="L94" class="blob-num js-line-number" data-line-number="94"></td>
+        <td id="LC94" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L95" class="blob-num js-line-number" data-line-number="95"></td>
+        <td id="LC95" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L96" class="blob-num js-line-number" data-line-number="96"></td>
+        <td id="LC96" class="blob-code blob-code-inner js-file-line">doch           |  (several meanings)</td>
+      </tr>
+      <tr>
+        <td id="L97" class="blob-num js-line-number" data-line-number="97"></td>
+        <td id="LC97" class="blob-code blob-code-inner js-file-line">dort           |  (over) there</td>
+      </tr>
+      <tr>
+        <td id="L98" class="blob-num js-line-number" data-line-number="98"></td>
+        <td id="LC98" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L99" class="blob-num js-line-number" data-line-number="99"></td>
+        <td id="LC99" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L100" class="blob-num js-line-number" data-line-number="100"></td>
+        <td id="LC100" class="blob-code blob-code-inner js-file-line">durch          |  through</td>
+      </tr>
+      <tr>
+        <td id="L101" class="blob-num js-line-number" data-line-number="101"></td>
+        <td id="LC101" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L102" class="blob-num js-line-number" data-line-number="102"></td>
+        <td id="LC102" class="blob-code blob-code-inner js-file-line">ein            |  a</td>
+      </tr>
+      <tr>
+        <td id="L103" class="blob-num js-line-number" data-line-number="103"></td>
+        <td id="LC103" class="blob-code blob-code-inner js-file-line">eine</td>
+      </tr>
+      <tr>
+        <td id="L104" class="blob-num js-line-number" data-line-number="104"></td>
+        <td id="LC104" class="blob-code blob-code-inner js-file-line">einem</td>
+      </tr>
+      <tr>
+        <td id="L105" class="blob-num js-line-number" data-line-number="105"></td>
+        <td id="LC105" class="blob-code blob-code-inner js-file-line">einen</td>
+      </tr>
+      <tr>
+        <td id="L106" class="blob-num js-line-number" data-line-number="106"></td>
+        <td id="LC106" class="blob-code blob-code-inner js-file-line">einer</td>
+      </tr>
+      <tr>
+        <td id="L107" class="blob-num js-line-number" data-line-number="107"></td>
+        <td id="LC107" class="blob-code blob-code-inner js-file-line">eines</td>
+      </tr>
+      <tr>
+        <td id="L108" class="blob-num js-line-number" data-line-number="108"></td>
+        <td id="LC108" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L109" class="blob-num js-line-number" data-line-number="109"></td>
+        <td id="LC109" class="blob-code blob-code-inner js-file-line">einig          |  some</td>
+      </tr>
+      <tr>
+        <td id="L110" class="blob-num js-line-number" data-line-number="110"></td>
+        <td id="LC110" class="blob-code blob-code-inner js-file-line">einige</td>
+      </tr>
+      <tr>
+        <td id="L111" class="blob-num js-line-number" data-line-number="111"></td>
+        <td id="LC111" class="blob-code blob-code-inner js-file-line">einigem</td>
+      </tr>
+      <tr>
+        <td id="L112" class="blob-num js-line-number" data-line-number="112"></td>
+        <td id="LC112" class="blob-code blob-code-inner js-file-line">einigen</td>
+      </tr>
+      <tr>
+        <td id="L113" class="blob-num js-line-number" data-line-number="113"></td>
+        <td id="LC113" class="blob-code blob-code-inner js-file-line">einiger</td>
+      </tr>
+      <tr>
+        <td id="L114" class="blob-num js-line-number" data-line-number="114"></td>
+        <td id="LC114" class="blob-code blob-code-inner js-file-line">einiges</td>
+      </tr>
+      <tr>
+        <td id="L115" class="blob-num js-line-number" data-line-number="115"></td>
+        <td id="LC115" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L116" class="blob-num js-line-number" data-line-number="116"></td>
+        <td id="LC116" class="blob-code blob-code-inner js-file-line">einmal         |  once</td>
+      </tr>
+      <tr>
+        <td id="L117" class="blob-num js-line-number" data-line-number="117"></td>
+        <td id="LC117" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L118" class="blob-num js-line-number" data-line-number="118"></td>
+        <td id="LC118" class="blob-code blob-code-inner js-file-line">er             |  he</td>
+      </tr>
+      <tr>
+        <td id="L119" class="blob-num js-line-number" data-line-number="119"></td>
+        <td id="LC119" class="blob-code blob-code-inner js-file-line">ihn            |  him</td>
+      </tr>
+      <tr>
+        <td id="L120" class="blob-num js-line-number" data-line-number="120"></td>
+        <td id="LC120" class="blob-code blob-code-inner js-file-line">ihm            |  to him</td>
+      </tr>
+      <tr>
+        <td id="L121" class="blob-num js-line-number" data-line-number="121"></td>
+        <td id="LC121" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L122" class="blob-num js-line-number" data-line-number="122"></td>
+        <td id="LC122" class="blob-code blob-code-inner js-file-line">es             |  it</td>
+      </tr>
+      <tr>
+        <td id="L123" class="blob-num js-line-number" data-line-number="123"></td>
+        <td id="LC123" class="blob-code blob-code-inner js-file-line">etwas          |  something</td>
+      </tr>
+      <tr>
+        <td id="L124" class="blob-num js-line-number" data-line-number="124"></td>
+        <td id="LC124" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L125" class="blob-num js-line-number" data-line-number="125"></td>
+        <td id="LC125" class="blob-code blob-code-inner js-file-line">euer           |  your</td>
+      </tr>
+      <tr>
+        <td id="L126" class="blob-num js-line-number" data-line-number="126"></td>
+        <td id="LC126" class="blob-code blob-code-inner js-file-line">eure</td>
+      </tr>
+      <tr>
+        <td id="L127" class="blob-num js-line-number" data-line-number="127"></td>
+        <td id="LC127" class="blob-code blob-code-inner js-file-line">eurem</td>
+      </tr>
+      <tr>
+        <td id="L128" class="blob-num js-line-number" data-line-number="128"></td>
+        <td id="LC128" class="blob-code blob-code-inner js-file-line">euren</td>
+      </tr>
+      <tr>
+        <td id="L129" class="blob-num js-line-number" data-line-number="129"></td>
+        <td id="LC129" class="blob-code blob-code-inner js-file-line">eurer</td>
+      </tr>
+      <tr>
+        <td id="L130" class="blob-num js-line-number" data-line-number="130"></td>
+        <td id="LC130" class="blob-code blob-code-inner js-file-line">eures</td>
+      </tr>
+      <tr>
+        <td id="L131" class="blob-num js-line-number" data-line-number="131"></td>
+        <td id="LC131" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L132" class="blob-num js-line-number" data-line-number="132"></td>
+        <td id="LC132" class="blob-code blob-code-inner js-file-line">für            |  for</td>
+      </tr>
+      <tr>
+        <td id="L133" class="blob-num js-line-number" data-line-number="133"></td>
+        <td id="LC133" class="blob-code blob-code-inner js-file-line">gegen          |  towards</td>
+      </tr>
+      <tr>
+        <td id="L134" class="blob-num js-line-number" data-line-number="134"></td>
+        <td id="LC134" class="blob-code blob-code-inner js-file-line">gewesen        |  p.p. of sein</td>
+      </tr>
+      <tr>
+        <td id="L135" class="blob-num js-line-number" data-line-number="135"></td>
+        <td id="LC135" class="blob-code blob-code-inner js-file-line">hab            |  have</td>
+      </tr>
+      <tr>
+        <td id="L136" class="blob-num js-line-number" data-line-number="136"></td>
+        <td id="LC136" class="blob-code blob-code-inner js-file-line">habe           |  have</td>
+      </tr>
+      <tr>
+        <td id="L137" class="blob-num js-line-number" data-line-number="137"></td>
+        <td id="LC137" class="blob-code blob-code-inner js-file-line">haben          |  have</td>
+      </tr>
+      <tr>
+        <td id="L138" class="blob-num js-line-number" data-line-number="138"></td>
+        <td id="LC138" class="blob-code blob-code-inner js-file-line">hat            |  has</td>
+      </tr>
+      <tr>
+        <td id="L139" class="blob-num js-line-number" data-line-number="139"></td>
+        <td id="LC139" class="blob-code blob-code-inner js-file-line">hatte          |  had</td>
+      </tr>
+      <tr>
+        <td id="L140" class="blob-num js-line-number" data-line-number="140"></td>
+        <td id="LC140" class="blob-code blob-code-inner js-file-line">hatten         |  had</td>
+      </tr>
+      <tr>
+        <td id="L141" class="blob-num js-line-number" data-line-number="141"></td>
+        <td id="LC141" class="blob-code blob-code-inner js-file-line">hier           |  here</td>
+      </tr>
+      <tr>
+        <td id="L142" class="blob-num js-line-number" data-line-number="142"></td>
+        <td id="LC142" class="blob-code blob-code-inner js-file-line">hin            |  there</td>
+      </tr>
+      <tr>
+        <td id="L143" class="blob-num js-line-number" data-line-number="143"></td>
+        <td id="LC143" class="blob-code blob-code-inner js-file-line">hinter         |  behind</td>
+      </tr>
+      <tr>
+        <td id="L144" class="blob-num js-line-number" data-line-number="144"></td>
+        <td id="LC144" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L145" class="blob-num js-line-number" data-line-number="145"></td>
+        <td id="LC145" class="blob-code blob-code-inner js-file-line">ich            |  I</td>
+      </tr>
+      <tr>
+        <td id="L146" class="blob-num js-line-number" data-line-number="146"></td>
+        <td id="LC146" class="blob-code blob-code-inner js-file-line">mich           |  me</td>
+      </tr>
+      <tr>
+        <td id="L147" class="blob-num js-line-number" data-line-number="147"></td>
+        <td id="LC147" class="blob-code blob-code-inner js-file-line">mir            |  to me</td>
+      </tr>
+      <tr>
+        <td id="L148" class="blob-num js-line-number" data-line-number="148"></td>
+        <td id="LC148" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L149" class="blob-num js-line-number" data-line-number="149"></td>
+        <td id="LC149" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L150" class="blob-num js-line-number" data-line-number="150"></td>
+        <td id="LC150" class="blob-code blob-code-inner js-file-line">ihr            |  you, to her</td>
+      </tr>
+      <tr>
+        <td id="L151" class="blob-num js-line-number" data-line-number="151"></td>
+        <td id="LC151" class="blob-code blob-code-inner js-file-line">ihre</td>
+      </tr>
+      <tr>
+        <td id="L152" class="blob-num js-line-number" data-line-number="152"></td>
+        <td id="LC152" class="blob-code blob-code-inner js-file-line">ihrem</td>
+      </tr>
+      <tr>
+        <td id="L153" class="blob-num js-line-number" data-line-number="153"></td>
+        <td id="LC153" class="blob-code blob-code-inner js-file-line">ihren</td>
+      </tr>
+      <tr>
+        <td id="L154" class="blob-num js-line-number" data-line-number="154"></td>
+        <td id="LC154" class="blob-code blob-code-inner js-file-line">ihrer</td>
+      </tr>
+      <tr>
+        <td id="L155" class="blob-num js-line-number" data-line-number="155"></td>
+        <td id="LC155" class="blob-code blob-code-inner js-file-line">ihres</td>
+      </tr>
+      <tr>
+        <td id="L156" class="blob-num js-line-number" data-line-number="156"></td>
+        <td id="LC156" class="blob-code blob-code-inner js-file-line">euch           |  to you</td>
+      </tr>
+      <tr>
+        <td id="L157" class="blob-num js-line-number" data-line-number="157"></td>
+        <td id="LC157" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L158" class="blob-num js-line-number" data-line-number="158"></td>
+        <td id="LC158" class="blob-code blob-code-inner js-file-line">im             |  in + dem</td>
+      </tr>
+      <tr>
+        <td id="L159" class="blob-num js-line-number" data-line-number="159"></td>
+        <td id="LC159" class="blob-code blob-code-inner js-file-line">in             |  in</td>
+      </tr>
+      <tr>
+        <td id="L160" class="blob-num js-line-number" data-line-number="160"></td>
+        <td id="LC160" class="blob-code blob-code-inner js-file-line">indem          |  while</td>
+      </tr>
+      <tr>
+        <td id="L161" class="blob-num js-line-number" data-line-number="161"></td>
+        <td id="LC161" class="blob-code blob-code-inner js-file-line">ins            |  in + das</td>
+      </tr>
+      <tr>
+        <td id="L162" class="blob-num js-line-number" data-line-number="162"></td>
+        <td id="LC162" class="blob-code blob-code-inner js-file-line">ist            |  is</td>
+      </tr>
+      <tr>
+        <td id="L163" class="blob-num js-line-number" data-line-number="163"></td>
+        <td id="LC163" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L164" class="blob-num js-line-number" data-line-number="164"></td>
+        <td id="LC164" class="blob-code blob-code-inner js-file-line">jede           |  each, every</td>
+      </tr>
+      <tr>
+        <td id="L165" class="blob-num js-line-number" data-line-number="165"></td>
+        <td id="LC165" class="blob-code blob-code-inner js-file-line">jedem</td>
+      </tr>
+      <tr>
+        <td id="L166" class="blob-num js-line-number" data-line-number="166"></td>
+        <td id="LC166" class="blob-code blob-code-inner js-file-line">jeden</td>
+      </tr>
+      <tr>
+        <td id="L167" class="blob-num js-line-number" data-line-number="167"></td>
+        <td id="LC167" class="blob-code blob-code-inner js-file-line">jeder</td>
+      </tr>
+      <tr>
+        <td id="L168" class="blob-num js-line-number" data-line-number="168"></td>
+        <td id="LC168" class="blob-code blob-code-inner js-file-line">jedes</td>
+      </tr>
+      <tr>
+        <td id="L169" class="blob-num js-line-number" data-line-number="169"></td>
+        <td id="LC169" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L170" class="blob-num js-line-number" data-line-number="170"></td>
+        <td id="LC170" class="blob-code blob-code-inner js-file-line">jene           |  that</td>
+      </tr>
+      <tr>
+        <td id="L171" class="blob-num js-line-number" data-line-number="171"></td>
+        <td id="LC171" class="blob-code blob-code-inner js-file-line">jenem</td>
+      </tr>
+      <tr>
+        <td id="L172" class="blob-num js-line-number" data-line-number="172"></td>
+        <td id="LC172" class="blob-code blob-code-inner js-file-line">jenen</td>
+      </tr>
+      <tr>
+        <td id="L173" class="blob-num js-line-number" data-line-number="173"></td>
+        <td id="LC173" class="blob-code blob-code-inner js-file-line">jener</td>
+      </tr>
+      <tr>
+        <td id="L174" class="blob-num js-line-number" data-line-number="174"></td>
+        <td id="LC174" class="blob-code blob-code-inner js-file-line">jenes</td>
+      </tr>
+      <tr>
+        <td id="L175" class="blob-num js-line-number" data-line-number="175"></td>
+        <td id="LC175" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L176" class="blob-num js-line-number" data-line-number="176"></td>
+        <td id="LC176" class="blob-code blob-code-inner js-file-line">jetzt          |  now</td>
+      </tr>
+      <tr>
+        <td id="L177" class="blob-num js-line-number" data-line-number="177"></td>
+        <td id="LC177" class="blob-code blob-code-inner js-file-line">kann           |  can</td>
+      </tr>
+      <tr>
+        <td id="L178" class="blob-num js-line-number" data-line-number="178"></td>
+        <td id="LC178" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L179" class="blob-num js-line-number" data-line-number="179"></td>
+        <td id="LC179" class="blob-code blob-code-inner js-file-line">kein           |  no</td>
+      </tr>
+      <tr>
+        <td id="L180" class="blob-num js-line-number" data-line-number="180"></td>
+        <td id="LC180" class="blob-code blob-code-inner js-file-line">keine</td>
+      </tr>
+      <tr>
+        <td id="L181" class="blob-num js-line-number" data-line-number="181"></td>
+        <td id="LC181" class="blob-code blob-code-inner js-file-line">keinem</td>
+      </tr>
+      <tr>
+        <td id="L182" class="blob-num js-line-number" data-line-number="182"></td>
+        <td id="LC182" class="blob-code blob-code-inner js-file-line">keinen</td>
+      </tr>
+      <tr>
+        <td id="L183" class="blob-num js-line-number" data-line-number="183"></td>
+        <td id="LC183" class="blob-code blob-code-inner js-file-line">keiner</td>
+      </tr>
+      <tr>
+        <td id="L184" class="blob-num js-line-number" data-line-number="184"></td>
+        <td id="LC184" class="blob-code blob-code-inner js-file-line">keines</td>
+      </tr>
+      <tr>
+        <td id="L185" class="blob-num js-line-number" data-line-number="185"></td>
+        <td id="LC185" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L186" class="blob-num js-line-number" data-line-number="186"></td>
+        <td id="LC186" class="blob-code blob-code-inner js-file-line">können         |  can</td>
+      </tr>
+      <tr>
+        <td id="L187" class="blob-num js-line-number" data-line-number="187"></td>
+        <td id="LC187" class="blob-code blob-code-inner js-file-line">könnte         |  could</td>
+      </tr>
+      <tr>
+        <td id="L188" class="blob-num js-line-number" data-line-number="188"></td>
+        <td id="LC188" class="blob-code blob-code-inner js-file-line">machen         |  do</td>
+      </tr>
+      <tr>
+        <td id="L189" class="blob-num js-line-number" data-line-number="189"></td>
+        <td id="LC189" class="blob-code blob-code-inner js-file-line">man            |  one</td>
+      </tr>
+      <tr>
+        <td id="L190" class="blob-num js-line-number" data-line-number="190"></td>
+        <td id="LC190" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L191" class="blob-num js-line-number" data-line-number="191"></td>
+        <td id="LC191" class="blob-code blob-code-inner js-file-line">manche         |  some, many a</td>
+      </tr>
+      <tr>
+        <td id="L192" class="blob-num js-line-number" data-line-number="192"></td>
+        <td id="LC192" class="blob-code blob-code-inner js-file-line">manchem</td>
+      </tr>
+      <tr>
+        <td id="L193" class="blob-num js-line-number" data-line-number="193"></td>
+        <td id="LC193" class="blob-code blob-code-inner js-file-line">manchen</td>
+      </tr>
+      <tr>
+        <td id="L194" class="blob-num js-line-number" data-line-number="194"></td>
+        <td id="LC194" class="blob-code blob-code-inner js-file-line">mancher</td>
+      </tr>
+      <tr>
+        <td id="L195" class="blob-num js-line-number" data-line-number="195"></td>
+        <td id="LC195" class="blob-code blob-code-inner js-file-line">manches</td>
+      </tr>
+      <tr>
+        <td id="L196" class="blob-num js-line-number" data-line-number="196"></td>
+        <td id="LC196" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L197" class="blob-num js-line-number" data-line-number="197"></td>
+        <td id="LC197" class="blob-code blob-code-inner js-file-line">mein           |  my</td>
+      </tr>
+      <tr>
+        <td id="L198" class="blob-num js-line-number" data-line-number="198"></td>
+        <td id="LC198" class="blob-code blob-code-inner js-file-line">meine</td>
+      </tr>
+      <tr>
+        <td id="L199" class="blob-num js-line-number" data-line-number="199"></td>
+        <td id="LC199" class="blob-code blob-code-inner js-file-line">meinem</td>
+      </tr>
+      <tr>
+        <td id="L200" class="blob-num js-line-number" data-line-number="200"></td>
+        <td id="LC200" class="blob-code blob-code-inner js-file-line">meinen</td>
+      </tr>
+      <tr>
+        <td id="L201" class="blob-num js-line-number" data-line-number="201"></td>
+        <td id="LC201" class="blob-code blob-code-inner js-file-line">meiner</td>
+      </tr>
+      <tr>
+        <td id="L202" class="blob-num js-line-number" data-line-number="202"></td>
+        <td id="LC202" class="blob-code blob-code-inner js-file-line">meines</td>
+      </tr>
+      <tr>
+        <td id="L203" class="blob-num js-line-number" data-line-number="203"></td>
+        <td id="LC203" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L204" class="blob-num js-line-number" data-line-number="204"></td>
+        <td id="LC204" class="blob-code blob-code-inner js-file-line">mit            |  with</td>
+      </tr>
+      <tr>
+        <td id="L205" class="blob-num js-line-number" data-line-number="205"></td>
+        <td id="LC205" class="blob-code blob-code-inner js-file-line">muss           |  must</td>
+      </tr>
+      <tr>
+        <td id="L206" class="blob-num js-line-number" data-line-number="206"></td>
+        <td id="LC206" class="blob-code blob-code-inner js-file-line">musste         |  had to</td>
+      </tr>
+      <tr>
+        <td id="L207" class="blob-num js-line-number" data-line-number="207"></td>
+        <td id="LC207" class="blob-code blob-code-inner js-file-line">nach           |  to(wards)</td>
+      </tr>
+      <tr>
+        <td id="L208" class="blob-num js-line-number" data-line-number="208"></td>
+        <td id="LC208" class="blob-code blob-code-inner js-file-line">nicht          |  not</td>
+      </tr>
+      <tr>
+        <td id="L209" class="blob-num js-line-number" data-line-number="209"></td>
+        <td id="LC209" class="blob-code blob-code-inner js-file-line">nichts         |  nothing</td>
+      </tr>
+      <tr>
+        <td id="L210" class="blob-num js-line-number" data-line-number="210"></td>
+        <td id="LC210" class="blob-code blob-code-inner js-file-line">noch           |  still, yet</td>
+      </tr>
+      <tr>
+        <td id="L211" class="blob-num js-line-number" data-line-number="211"></td>
+        <td id="LC211" class="blob-code blob-code-inner js-file-line">nun            |  now</td>
+      </tr>
+      <tr>
+        <td id="L212" class="blob-num js-line-number" data-line-number="212"></td>
+        <td id="LC212" class="blob-code blob-code-inner js-file-line">nur            |  only</td>
+      </tr>
+      <tr>
+        <td id="L213" class="blob-num js-line-number" data-line-number="213"></td>
+        <td id="LC213" class="blob-code blob-code-inner js-file-line">ob             |  whether</td>
+      </tr>
+      <tr>
+        <td id="L214" class="blob-num js-line-number" data-line-number="214"></td>
+        <td id="LC214" class="blob-code blob-code-inner js-file-line">oder           |  or</td>
+      </tr>
+      <tr>
+        <td id="L215" class="blob-num js-line-number" data-line-number="215"></td>
+        <td id="LC215" class="blob-code blob-code-inner js-file-line">ohne           |  without</td>
+      </tr>
+      <tr>
+        <td id="L216" class="blob-num js-line-number" data-line-number="216"></td>
+        <td id="LC216" class="blob-code blob-code-inner js-file-line">sehr           |  very</td>
+      </tr>
+      <tr>
+        <td id="L217" class="blob-num js-line-number" data-line-number="217"></td>
+        <td id="LC217" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L218" class="blob-num js-line-number" data-line-number="218"></td>
+        <td id="LC218" class="blob-code blob-code-inner js-file-line">sein           |  his</td>
+      </tr>
+      <tr>
+        <td id="L219" class="blob-num js-line-number" data-line-number="219"></td>
+        <td id="LC219" class="blob-code blob-code-inner js-file-line">seine</td>
+      </tr>
+      <tr>
+        <td id="L220" class="blob-num js-line-number" data-line-number="220"></td>
+        <td id="LC220" class="blob-code blob-code-inner js-file-line">seinem</td>
+      </tr>
+      <tr>
+        <td id="L221" class="blob-num js-line-number" data-line-number="221"></td>
+        <td id="LC221" class="blob-code blob-code-inner js-file-line">seinen</td>
+      </tr>
+      <tr>
+        <td id="L222" class="blob-num js-line-number" data-line-number="222"></td>
+        <td id="LC222" class="blob-code blob-code-inner js-file-line">seiner</td>
+      </tr>
+      <tr>
+        <td id="L223" class="blob-num js-line-number" data-line-number="223"></td>
+        <td id="LC223" class="blob-code blob-code-inner js-file-line">seines</td>
+      </tr>
+      <tr>
+        <td id="L224" class="blob-num js-line-number" data-line-number="224"></td>
+        <td id="LC224" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L225" class="blob-num js-line-number" data-line-number="225"></td>
+        <td id="LC225" class="blob-code blob-code-inner js-file-line">selbst         |  self</td>
+      </tr>
+      <tr>
+        <td id="L226" class="blob-num js-line-number" data-line-number="226"></td>
+        <td id="LC226" class="blob-code blob-code-inner js-file-line">sich           |  herself</td>
+      </tr>
+      <tr>
+        <td id="L227" class="blob-num js-line-number" data-line-number="227"></td>
+        <td id="LC227" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L228" class="blob-num js-line-number" data-line-number="228"></td>
+        <td id="LC228" class="blob-code blob-code-inner js-file-line">sie            |  they, she</td>
+      </tr>
+      <tr>
+        <td id="L229" class="blob-num js-line-number" data-line-number="229"></td>
+        <td id="LC229" class="blob-code blob-code-inner js-file-line">ihnen          |  to them</td>
+      </tr>
+      <tr>
+        <td id="L230" class="blob-num js-line-number" data-line-number="230"></td>
+        <td id="LC230" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L231" class="blob-num js-line-number" data-line-number="231"></td>
+        <td id="LC231" class="blob-code blob-code-inner js-file-line">sind           |  are</td>
+      </tr>
+      <tr>
+        <td id="L232" class="blob-num js-line-number" data-line-number="232"></td>
+        <td id="LC232" class="blob-code blob-code-inner js-file-line">so             |  so</td>
+      </tr>
+      <tr>
+        <td id="L233" class="blob-num js-line-number" data-line-number="233"></td>
+        <td id="LC233" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L234" class="blob-num js-line-number" data-line-number="234"></td>
+        <td id="LC234" class="blob-code blob-code-inner js-file-line">solche         |  such</td>
+      </tr>
+      <tr>
+        <td id="L235" class="blob-num js-line-number" data-line-number="235"></td>
+        <td id="LC235" class="blob-code blob-code-inner js-file-line">solchem</td>
+      </tr>
+      <tr>
+        <td id="L236" class="blob-num js-line-number" data-line-number="236"></td>
+        <td id="LC236" class="blob-code blob-code-inner js-file-line">solchen</td>
+      </tr>
+      <tr>
+        <td id="L237" class="blob-num js-line-number" data-line-number="237"></td>
+        <td id="LC237" class="blob-code blob-code-inner js-file-line">solcher</td>
+      </tr>
+      <tr>
+        <td id="L238" class="blob-num js-line-number" data-line-number="238"></td>
+        <td id="LC238" class="blob-code blob-code-inner js-file-line">solches</td>
+      </tr>
+      <tr>
+        <td id="L239" class="blob-num js-line-number" data-line-number="239"></td>
+        <td id="LC239" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L240" class="blob-num js-line-number" data-line-number="240"></td>
+        <td id="LC240" class="blob-code blob-code-inner js-file-line">soll           |  shall</td>
+      </tr>
+      <tr>
+        <td id="L241" class="blob-num js-line-number" data-line-number="241"></td>
+        <td id="LC241" class="blob-code blob-code-inner js-file-line">sollte         |  should</td>
+      </tr>
+      <tr>
+        <td id="L242" class="blob-num js-line-number" data-line-number="242"></td>
+        <td id="LC242" class="blob-code blob-code-inner js-file-line">sondern        |  but</td>
+      </tr>
+      <tr>
+        <td id="L243" class="blob-num js-line-number" data-line-number="243"></td>
+        <td id="LC243" class="blob-code blob-code-inner js-file-line">sonst          |  else</td>
+      </tr>
+      <tr>
+        <td id="L244" class="blob-num js-line-number" data-line-number="244"></td>
+        <td id="LC244" class="blob-code blob-code-inner js-file-line">über           |  over</td>
+      </tr>
+      <tr>
+        <td id="L245" class="blob-num js-line-number" data-line-number="245"></td>
+        <td id="LC245" class="blob-code blob-code-inner js-file-line">um             |  about, around</td>
+      </tr>
+      <tr>
+        <td id="L246" class="blob-num js-line-number" data-line-number="246"></td>
+        <td id="LC246" class="blob-code blob-code-inner js-file-line">und            |  and</td>
+      </tr>
+      <tr>
+        <td id="L247" class="blob-num js-line-number" data-line-number="247"></td>
+        <td id="LC247" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L248" class="blob-num js-line-number" data-line-number="248"></td>
+        <td id="LC248" class="blob-code blob-code-inner js-file-line">uns            |  us</td>
+      </tr>
+      <tr>
+        <td id="L249" class="blob-num js-line-number" data-line-number="249"></td>
+        <td id="LC249" class="blob-code blob-code-inner js-file-line">unse</td>
+      </tr>
+      <tr>
+        <td id="L250" class="blob-num js-line-number" data-line-number="250"></td>
+        <td id="LC250" class="blob-code blob-code-inner js-file-line">unsem</td>
+      </tr>
+      <tr>
+        <td id="L251" class="blob-num js-line-number" data-line-number="251"></td>
+        <td id="LC251" class="blob-code blob-code-inner js-file-line">unsen</td>
+      </tr>
+      <tr>
+        <td id="L252" class="blob-num js-line-number" data-line-number="252"></td>
+        <td id="LC252" class="blob-code blob-code-inner js-file-line">unser</td>
+      </tr>
+      <tr>
+        <td id="L253" class="blob-num js-line-number" data-line-number="253"></td>
+        <td id="LC253" class="blob-code blob-code-inner js-file-line">unses</td>
+      </tr>
+      <tr>
+        <td id="L254" class="blob-num js-line-number" data-line-number="254"></td>
+        <td id="LC254" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L255" class="blob-num js-line-number" data-line-number="255"></td>
+        <td id="LC255" class="blob-code blob-code-inner js-file-line">unter          |  under</td>
+      </tr>
+      <tr>
+        <td id="L256" class="blob-num js-line-number" data-line-number="256"></td>
+        <td id="LC256" class="blob-code blob-code-inner js-file-line">viel           |  much</td>
+      </tr>
+      <tr>
+        <td id="L257" class="blob-num js-line-number" data-line-number="257"></td>
+        <td id="LC257" class="blob-code blob-code-inner js-file-line">vom            |  von + dem</td>
+      </tr>
+      <tr>
+        <td id="L258" class="blob-num js-line-number" data-line-number="258"></td>
+        <td id="LC258" class="blob-code blob-code-inner js-file-line">von            |  from</td>
+      </tr>
+      <tr>
+        <td id="L259" class="blob-num js-line-number" data-line-number="259"></td>
+        <td id="LC259" class="blob-code blob-code-inner js-file-line">vor            |  before</td>
+      </tr>
+      <tr>
+        <td id="L260" class="blob-num js-line-number" data-line-number="260"></td>
+        <td id="LC260" class="blob-code blob-code-inner js-file-line">während        |  while</td>
+      </tr>
+      <tr>
+        <td id="L261" class="blob-num js-line-number" data-line-number="261"></td>
+        <td id="LC261" class="blob-code blob-code-inner js-file-line">war            |  was</td>
+      </tr>
+      <tr>
+        <td id="L262" class="blob-num js-line-number" data-line-number="262"></td>
+        <td id="LC262" class="blob-code blob-code-inner js-file-line">waren          |  were</td>
+      </tr>
+      <tr>
+        <td id="L263" class="blob-num js-line-number" data-line-number="263"></td>
+        <td id="LC263" class="blob-code blob-code-inner js-file-line">warst          |  wast</td>
+      </tr>
+      <tr>
+        <td id="L264" class="blob-num js-line-number" data-line-number="264"></td>
+        <td id="LC264" class="blob-code blob-code-inner js-file-line">was            |  what</td>
+      </tr>
+      <tr>
+        <td id="L265" class="blob-num js-line-number" data-line-number="265"></td>
+        <td id="LC265" class="blob-code blob-code-inner js-file-line">weg            |  away, off</td>
+      </tr>
+      <tr>
+        <td id="L266" class="blob-num js-line-number" data-line-number="266"></td>
+        <td id="LC266" class="blob-code blob-code-inner js-file-line">weil           |  because</td>
+      </tr>
+      <tr>
+        <td id="L267" class="blob-num js-line-number" data-line-number="267"></td>
+        <td id="LC267" class="blob-code blob-code-inner js-file-line">weiter         |  further</td>
+      </tr>
+      <tr>
+        <td id="L268" class="blob-num js-line-number" data-line-number="268"></td>
+        <td id="LC268" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L269" class="blob-num js-line-number" data-line-number="269"></td>
+        <td id="LC269" class="blob-code blob-code-inner js-file-line">welche         |  which</td>
+      </tr>
+      <tr>
+        <td id="L270" class="blob-num js-line-number" data-line-number="270"></td>
+        <td id="LC270" class="blob-code blob-code-inner js-file-line">welchem</td>
+      </tr>
+      <tr>
+        <td id="L271" class="blob-num js-line-number" data-line-number="271"></td>
+        <td id="LC271" class="blob-code blob-code-inner js-file-line">welchen</td>
+      </tr>
+      <tr>
+        <td id="L272" class="blob-num js-line-number" data-line-number="272"></td>
+        <td id="LC272" class="blob-code blob-code-inner js-file-line">welcher</td>
+      </tr>
+      <tr>
+        <td id="L273" class="blob-num js-line-number" data-line-number="273"></td>
+        <td id="LC273" class="blob-code blob-code-inner js-file-line">welches</td>
+      </tr>
+      <tr>
+        <td id="L274" class="blob-num js-line-number" data-line-number="274"></td>
+        <td id="LC274" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+      <tr>
+        <td id="L275" class="blob-num js-line-number" data-line-number="275"></td>
+        <td id="LC275" class="blob-code blob-code-inner js-file-line">wenn           |  when</td>
+      </tr>
+      <tr>
+        <td id="L276" class="blob-num js-line-number" data-line-number="276"></td>
+        <td id="LC276" class="blob-code blob-code-inner js-file-line">werde          |  will</td>
+      </tr>
+      <tr>
+        <td id="L277" class="blob-num js-line-number" data-line-number="277"></td>
+        <td id="LC277" class="blob-code blob-code-inner js-file-line">werden         |  will</td>
+      </tr>
+      <tr>
+        <td id="L278" class="blob-num js-line-number" data-line-number="278"></td>
+        <td id="LC278" class="blob-code blob-code-inner js-file-line">wie            |  how</td>
+      </tr>
+      <tr>
+        <td id="L279" class="blob-num js-line-number" data-line-number="279"></td>
+        <td id="LC279" class="blob-code blob-code-inner js-file-line">wieder         |  again</td>
+      </tr>
+      <tr>
+        <td id="L280" class="blob-num js-line-number" data-line-number="280"></td>
+        <td id="LC280" class="blob-code blob-code-inner js-file-line">will           |  want</td>
+      </tr>
+      <tr>
+        <td id="L281" class="blob-num js-line-number" data-line-number="281"></td>
+        <td id="LC281" class="blob-code blob-code-inner js-file-line">wir            |  we</td>
+      </tr>
+      <tr>
+        <td id="L282" class="blob-num js-line-number" data-line-number="282"></td>
+        <td id="LC282" class="blob-code blob-code-inner js-file-line">wird           |  will</td>
+      </tr>
+      <tr>
+        <td id="L283" class="blob-num js-line-number" data-line-number="283"></td>
+        <td id="LC283" class="blob-code blob-code-inner js-file-line">wirst          |  willst</td>
+      </tr>
+      <tr>
+        <td id="L284" class="blob-num js-line-number" data-line-number="284"></td>
+        <td id="LC284" class="blob-code blob-code-inner js-file-line">wo             |  where</td>
+      </tr>
+      <tr>
+        <td id="L285" class="blob-num js-line-number" data-line-number="285"></td>
+        <td id="LC285" class="blob-code blob-code-inner js-file-line">wollen         |  want</td>
+      </tr>
+      <tr>
+        <td id="L286" class="blob-num js-line-number" data-line-number="286"></td>
+        <td id="LC286" class="blob-code blob-code-inner js-file-line">wollte         |  wanted</td>
+      </tr>
+      <tr>
+        <td id="L287" class="blob-num js-line-number" data-line-number="287"></td>
+        <td id="LC287" class="blob-code blob-code-inner js-file-line">würde          |  would</td>
+      </tr>
+      <tr>
+        <td id="L288" class="blob-num js-line-number" data-line-number="288"></td>
+        <td id="LC288" class="blob-code blob-code-inner js-file-line">würden         |  would</td>
+      </tr>
+      <tr>
+        <td id="L289" class="blob-num js-line-number" data-line-number="289"></td>
+        <td id="LC289" class="blob-code blob-code-inner js-file-line">zu             |  to</td>
+      </tr>
+      <tr>
+        <td id="L290" class="blob-num js-line-number" data-line-number="290"></td>
+        <td id="LC290" class="blob-code blob-code-inner js-file-line">zum            |  zu + dem</td>
+      </tr>
+      <tr>
+        <td id="L291" class="blob-num js-line-number" data-line-number="291"></td>
+        <td id="LC291" class="blob-code blob-code-inner js-file-line">zur            |  zu + der</td>
+      </tr>
+      <tr>
+        <td id="L292" class="blob-num js-line-number" data-line-number="292"></td>
+        <td id="LC292" class="blob-code blob-code-inner js-file-line">zwar           |  indeed</td>
+      </tr>
+      <tr>
+        <td id="L293" class="blob-num js-line-number" data-line-number="293"></td>
+        <td id="LC293" class="blob-code blob-code-inner js-file-line">zwischen       |  between</td>
+      </tr>
+      <tr>
+        <td id="L294" class="blob-num js-line-number" data-line-number="294"></td>
+        <td id="LC294" class="blob-code blob-code-inner js-file-line">
+</td>
+      </tr>
+</table>
+
+  </div>
+
+</div>
+
+<button type="button" data-facebox="#jump-to-line" data-facebox-class="linejump" data-hotkey="l" class="hidden">Jump to Line</button>
+<div id="jump-to-line" style="display:none">
+  <!-- </textarea> --><!-- '"` --><form accept-charset="UTF-8" action="" class="js-jump-to-line-form" method="get"><div style="margin:0;padding:0;display:inline"><input name="utf8" type="hidden" value="&#x2713;" /></div>
+    <input class="linejump-input js-jump-to-line-field" type="text" placeholder="Jump to line&hellip;" aria-label="Jump to line" autofocus>
+    <button type="submit" class="btn">Go</button>
+</form></div>
+
+  </div>
+  <div class="modal-backdrop"></div>
+</div>
+
+
+    </div>
+  </div>
+
+    </div>
+
+        <div class="container site-footer-container">
+  <div class="site-footer" role="contentinfo">
+    <ul class="site-footer-links right">
+        <li><a href="https://status.github.com/" data-ga-click="Footer, go to status, text:status">Status</a></li>
+      <li><a href="https://developer.github.com" data-ga-click="Footer, go to api, text:api">API</a></li>
+      <li><a href="https://training.github.com" data-ga-click="Footer, go to training, text:training">Training</a></li>
+      <li><a href="https://shop.github.com" data-ga-click="Footer, go to shop, text:shop">Shop</a></li>
+        <li><a href="https://github.com/blog" data-ga-click="Footer, go to blog, text:blog">Blog</a></li>
+        <li><a href="https://github.com/about" data-ga-click="Footer, go to about, text:about">About</a></li>
+          <li><a href="https://github.com/pricing" data-ga-click="Footer, go to pricing, text:pricing">Pricing</a></li>
+
+    </ul>
+
+    <a href="https://github.com" aria-label="Homepage" class="site-footer-mark">
+      <svg aria-hidden="true" class="octicon octicon-mark-github" height="24" role="img" title="GitHub " version="1.1" viewBox="0 0 16 16" width="24"><path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59 0.4 0.07 0.55-0.17 0.55-0.38 0-0.19-0.01-0.82-0.01-1.49-2.01 0.37-2.53-0.49-2.69-0.94-0.09-0.23-0.48-0.94-0.82-1.13-0.28-0.15-0.68-0.52-0.01-0.53 0.63-0.01 1.08 0.58 1.23 0.82 0.72 1.21 1.87 0.87 2.33 0.66 0.07-0.52 0.28-0.87 0.51-1.07-1.78-0.2-3.64-0.89-3.64-3.95 0-0.87 0.31-1.59 0.82-2.15-0.08-0.2-0.36-1.02 0.08-2.12 0 0 0.67-0.21 2.2 0.82 0.64-0.18 1.32-0.27 2-0.27 0.68 0 1.36 0.09 2 0.27 1.53-1.04 2.2-0.82 2.2-0.82 0.44 1.1 0.16 1.92 0.08 2.12 0.51 0.56 0.82 1.27 0.82 2.15 0 3.07-1.87 3.75-3.65 3.95 0.29 0.25 0.54 0.73 0.54 1.48 0 1.07-0.01 1.93-0.01 2.2 0 0.21 0.15 0.46 0.55 0.38C13.71 14.53 16 11.53 16 8 16 3.58 12.42 0 8 0z"></path></svg>
+</a>
+    <ul class="site-footer-links">
+      <li>&copy; 2016 <span title="0.13439s from github-fe127-cp1-prd.iad.github.net">GitHub</span>, Inc.</li>
+        <li><a href="https://github.com/site/terms" data-ga-click="Footer, go to terms, text:terms">Terms</a></li>
+        <li><a href="https://github.com/site/privacy" data-ga-click="Footer, go to privacy, text:privacy">Privacy</a></li>
+        <li><a href="https://github.com/security" data-ga-click="Footer, go to security, text:security">Security</a></li>
+        <li><a href="https://github.com/contact" data-ga-click="Footer, go to contact, text:contact">Contact</a></li>
+        <li><a href="https://help.github.com" data-ga-click="Footer, go to help, text:help">Help</a></li>
+    </ul>
+  </div>
+</div>
+
+
+
+    
+    
+    
+
+    <div id="ajax-error-message" class="ajax-error-message flash flash-error">
+      <svg aria-hidden="true" class="octicon octicon-alert" height="16" role="img" version="1.1" viewBox="0 0 16 16" width="16"><path d="M15.72 12.5l-6.85-11.98C8.69 0.21 8.36 0.02 8 0.02s-0.69 0.19-0.87 0.5l-6.85 11.98c-0.18 0.31-0.18 0.69 0 1C0.47 13.81 0.8 14 1.15 14h13.7c0.36 0 0.69-0.19 0.86-0.5S15.89 12.81 15.72 12.5zM9 12H7V10h2V12zM9 9H7V5h2V9z"></path></svg>
+      <button type="button" class="flash-close js-flash-close js-ajax-error-dismiss" aria-label="Dismiss error">
+        <svg aria-hidden="true" class="octicon octicon-x" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M7.48 8l3.75 3.75-1.48 1.48-3.75-3.75-3.75 3.75-1.48-1.48 3.75-3.75L0.77 4.25l1.48-1.48 3.75 3.75 3.75-3.75 1.48 1.48-3.75 3.75z"></path></svg>
+      </button>
+      Something went wrong with that request. Please try again.
+    </div>
+
+
+      <script crossorigin="anonymous" src="https://assets-cdn.github.com/assets/compat-7db58f8b7b91111107fac755dd8b178fe7db0f209ced51fc339c446ad3f8da2b.js"></script>
+      <script crossorigin="anonymous" src="https://assets-cdn.github.com/assets/frameworks-96e29b697be2d5e4c062e40ce2dab35f90b4cd378cfbbcd6dd51a3f280e7d426.js"></script>
+      <script async="async" crossorigin="anonymous" src="https://assets-cdn.github.com/assets/github-061f27987895ca9e650b5c852acbea14f7b6e360f8d0a2bf2e5692911aa3926c.js"></script>
+      
+      
+      
+    <div class="js-stale-session-flash stale-session-flash flash flash-warn flash-banner hidden">
+      <svg aria-hidden="true" class="octicon octicon-alert" height="16" role="img" version="1.1" viewBox="0 0 16 16" width="16"><path d="M15.72 12.5l-6.85-11.98C8.69 0.21 8.36 0.02 8 0.02s-0.69 0.19-0.87 0.5l-6.85 11.98c-0.18 0.31-0.18 0.69 0 1C0.47 13.81 0.8 14 1.15 14h13.7c0.36 0 0.69-0.19 0.86-0.5S15.89 12.81 15.72 12.5zM9 12H7V10h2V12zM9 9H7V5h2V9z"></path></svg>
+      <span class="signed-in-tab-flash">You signed in with another tab or window. <a href="">Reload</a> to refresh your session.</span>
+      <span class="signed-out-tab-flash">You signed out in another tab or window. <a href="">Reload</a> to refresh your session.</span>
+    </div>
+    <div class="facebox" id="facebox" style="display:none;">
+  <div class="facebox-popup">
+    <div class="facebox-content" role="dialog" aria-labelledby="facebox-header" aria-describedby="facebox-description">
+    </div>
+    <button type="button" class="facebox-close js-facebox-close" aria-label="Close modal">
+      <svg aria-hidden="true" class="octicon octicon-x" height="16" role="img" version="1.1" viewBox="0 0 12 16" width="12"><path d="M7.48 8l3.75 3.75-1.48 1.48-3.75-3.75-3.75 3.75-1.48-1.48 3.75-3.75L0.77 4.25l1.48-1.48 3.75 3.75 3.75-3.75 1.48 1.48-3.75 3.75z"></path></svg>
+    </button>
+  </div>
+</div>
+
+  </body>
+</html>
+
diff --git a/src/lib-fts/test-fts-filter.c b/src/lib-fts/test-fts-filter.c
index 55dd25f640e..a3844b477a3 100644
--- a/src/lib-fts/test-fts-filter.c
+++ b/src/lib-fts/test-fts-filter.c
@@ -348,6 +348,22 @@ static void test_fts_filter_stopwords_fail_lazy_init(void)
 
 }
 
+static void test_fts_filter_stopwords_malformed(void)
+{
+	const struct fts_language malformed = { .name = "malformed" };
+	struct fts_filter *filter = NULL;
+	const char *error = NULL, *token = "foobar";
+
+	test_begin("fts filter stopwords, malformed list");
+	test_assert(fts_filter_create(fts_filter_stopwords, NULL, &malformed, stopword_settings, &filter, &error) == 0);
+	test_expect_error_string("seems empty. Is the file correctly formatted?");
+	test_assert(fts_filter_filter(filter, &token, &error) > 0);
+	test_expect_no_more_errors();
+	fts_filter_unref(&filter);
+	test_end();
+
+}
+
 #ifdef HAVE_FTS_STEMMER
 static void test_fts_filter_stemmer_snowball_stem_english(void)
 {
@@ -893,6 +909,7 @@ int main(void)
 		test_fts_filter_stopwords_fra,
 		test_fts_filter_stopwords_no,
 		test_fts_filter_stopwords_fail_lazy_init,
+		test_fts_filter_stopwords_malformed,
 #ifdef HAVE_FTS_STEMMER
 		test_fts_filter_stemmer_snowball_stem_english,
 		test_fts_filter_stemmer_snowball_stem_french,

From b89dd50a257a179b2f8479ba0618d322127db971 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Wed, 27 Apr 2016 09:14:29 +0300
Subject: [PATCH 223/450] auth: Add PBKDF2 password scheme

---
 src/auth/Makefile.am              |  3 +-
 src/auth/password-scheme-pbkdf2.c | 82 +++++++++++++++++++++++++++++++
 src/auth/password-scheme.c        |  3 +-
 src/auth/password-scheme.h        |  5 ++
 4 files changed, 91 insertions(+), 2 deletions(-)
 create mode 100644 src/auth/password-scheme-pbkdf2.c

diff --git a/src/auth/Makefile.am b/src/auth/Makefile.am
index 0fbc56de0bf..574b854899a 100644
--- a/src/auth/Makefile.am
+++ b/src/auth/Makefile.am
@@ -47,7 +47,8 @@ libpassword_a_SOURCES = \
 	password-scheme-md5crypt.c \
 	password-scheme-scram.c \
 	password-scheme-otp.c \
-	password-scheme-rpa.c
+	password-scheme-rpa.c \
+	password-scheme-pbkdf2.c
 
 auth_libs = \
 	libstats_auth.la \
diff --git a/src/auth/password-scheme-pbkdf2.c b/src/auth/password-scheme-pbkdf2.c
new file mode 100644
index 00000000000..36305881923
--- /dev/null
+++ b/src/auth/password-scheme-pbkdf2.c
@@ -0,0 +1,82 @@
+/* Copyright (c) 2015 Dovecot Oy, see the included COPYING file */
+
+#include "lib.h"
+#include "buffer.h"
+#include "str.h"
+#include "password-scheme.h"
+#include "hex-binary.h"
+#include "hash-method.h"
+#include "pkcs5.h"
+
+#define PBKDF2_KEY_SIZE_SHA1 20
+
+#define PBKDF2_GENERATE_SALT_LEN      16
+#define PBKDF2_ROUNDS_DEFAULT          5000
+
+static void
+pbkdf_run(const char *plaintext, const char *salt,
+	  unsigned int rounds, unsigned char key_r[PBKDF2_KEY_SIZE_SHA1])
+{
+	memset(key_r, 0, PBKDF2_KEY_SIZE_SHA1);
+	buffer_t buf;
+	buffer_create_from_data(&buf, key_r, PBKDF2_KEY_SIZE_SHA1);
+
+	pkcs5_pbkdf(PKCS5_PBKDF2, hash_method_lookup("sha1"),
+		(const unsigned char *)plaintext, strlen(plaintext),
+		(const unsigned char *)salt, strlen(salt),
+		rounds, PBKDF2_KEY_SIZE_SHA1, &buf);
+}
+
+void pbkdf2_generate(const char *plaintext, const char *user ATTR_UNUSED,
+		const unsigned char **raw_password_r, size_t *size_r)
+{
+	unsigned char key[PBKDF2_KEY_SIZE_SHA1];
+	const char *salt;
+	string_t *str = t_str_new(64);
+	unsigned int rounds = password_scheme_encryption_rounds;
+
+	if (rounds == 0)
+		rounds = PBKDF2_ROUNDS_DEFAULT;
+	salt = password_generate_salt(PBKDF2_GENERATE_SALT_LEN);
+	pbkdf_run(plaintext, salt, rounds, key);
+
+	str_printfa(str, "$1$%s$%u$", salt, rounds);
+	binary_to_hex_append(str, key, sizeof(key));
+
+	*raw_password_r = str_data(str);
+	*size_r = str_len(str);
+}
+
+int pbkdf2_verify(const char *plaintext, const char *user ATTR_UNUSED,
+	      const unsigned char *raw_password, size_t size,
+	      const char **error_r)
+{
+	const char *const *fields;
+	const char *salt;
+	unsigned int rounds;
+	unsigned char key1[PBKDF2_KEY_SIZE_SHA1], key2[PBKDF2_KEY_SIZE_SHA1];
+	buffer_t buf;
+
+	/* $1$salt$rounds$hash */
+	if (size < 3 || memcmp(raw_password, "$1$", 3) != 0) {
+		*error_r = "Invalid PBKDF2 passdb entry prefix";
+		return -1;
+	}
+
+	fields = t_strsplit(t_strndup(raw_password + 3, size - 3), "$");
+	salt = fields[0];
+	if (str_array_length(fields) != 3 ||
+	    str_to_uint(fields[1], &rounds) < 0) {
+		*error_r = "Invalid PBKDF2 passdb entry format";
+		return -1;
+	}
+	buffer_create_from_data(&buf, key1, sizeof(key1));
+	if (strlen(fields[2]) != sizeof(key1)*2 ||
+	    hex_to_binary(fields[2], &buf) < 0) {
+		*error_r = "PBKDF2 hash not 160bit hex-encoded";
+		return -1;
+	}
+
+	pbkdf_run(plaintext, salt, rounds, key2);
+	return memcmp(key1, key2, sizeof(key1)) == 0 ? 1 : 0;
+}
diff --git a/src/auth/password-scheme.c b/src/auth/password-scheme.c
index baefb9532cb..1633e50b1ea 100644
--- a/src/auth/password-scheme.c
+++ b/src/auth/password-scheme.c
@@ -825,7 +825,8 @@ static const struct password_scheme builtin_schemes[] = {
 	{ "NTLM", PW_ENCODING_HEX, NTLMSSP_HASH_SIZE, NULL, ntlm_generate },
 	{ "OTP", PW_ENCODING_NONE, 0, otp_verify, otp_generate },
 	{ "SKEY", PW_ENCODING_NONE, 0, otp_verify, skey_generate },
-	{ "RPA", PW_ENCODING_HEX, MD5_RESULTLEN, NULL, rpa_generate }
+	{ "RPA", PW_ENCODING_HEX, MD5_RESULTLEN, NULL, rpa_generate },
+        { "PBKDF2", PW_ENCODING_NONE, 0, pbkdf2_verify, pbkdf2_generate },
 };
 
 void password_scheme_register(const struct password_scheme *scheme)
diff --git a/src/auth/password-scheme.h b/src/auth/password-scheme.h
index 438e849919d..65a92ef1c5e 100644
--- a/src/auth/password-scheme.h
+++ b/src/auth/password-scheme.h
@@ -95,6 +95,11 @@ int scram_sha1_verify(const char *plaintext, const char *user ATTR_UNUSED,
 		      const char **error_r ATTR_UNUSED);
 void scram_sha1_generate(const char *plaintext, const char *user ATTR_UNUSED,
 			 const unsigned char **raw_password_r, size_t *size_r);
+void pbkdf2_generate(const char *plaintext, const char *user ATTR_UNUSED,
+		     const unsigned char **raw_password_r, size_t *size_r);
+int pbkdf2_verify(const char *plaintext, const char *user ATTR_UNUSED,
+		  const unsigned char *raw_password, size_t size,
+		  const char **error_r);
 
 /* check wich of the algorithms Blowfisch, SHA-256 and SHA-512 are
    supported by the used libc's/glibc's crypt() */

From 71e834420671e053f994a46673392cf88a7f5772 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Wed, 20 Apr 2016 17:34:53 +0300
Subject: [PATCH 224/450] lib: Add PKCS#5 pbkdf1 and 2

---
 src/lib/Makefile.am  |  3 ++
 src/lib/pkcs5.c      | 95 ++++++++++++++++++++++++++++++++++++++++++++
 src/lib/pkcs5.h      | 35 ++++++++++++++++
 src/lib/test-lib.c   |  1 +
 src/lib/test-lib.h   |  1 +
 src/lib/test-pkcs5.c | 49 +++++++++++++++++++++++
 6 files changed, 184 insertions(+)
 create mode 100644 src/lib/pkcs5.c
 create mode 100644 src/lib/pkcs5.h
 create mode 100644 src/lib/test-pkcs5.c

diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am
index d5705e90587..7845ffe5f64 100644
--- a/src/lib/Makefile.am
+++ b/src/lib/Makefile.am
@@ -114,6 +114,7 @@ liblib_la_SOURCES = \
 	ostream-hash.c \
 	ostream-null.c \
 	ostream-rawlog.c \
+	pkcs5.c \
 	primes.c \
 	printf-format-fix.c \
 	process-title.c \
@@ -247,6 +248,7 @@ headers = \
 	ostream-private.h \
 	ostream-null.h \
 	ostream-rawlog.h \
+	pkcs5.h \
 	primes.h \
 	printf-format-fix.h \
 	process-title.h \
@@ -328,6 +330,7 @@ test_lib_SOURCES = \
 	test-json-tree.c \
 	test-llist.c \
 	test-mempool-alloconly.c \
+	test-pkcs5.c \
 	test-net.c \
 	test-numpack.c \
 	test-ostream-escaped.c \
diff --git a/src/lib/pkcs5.c b/src/lib/pkcs5.c
new file mode 100644
index 00000000000..449b1e88e24
--- /dev/null
+++ b/src/lib/pkcs5.c
@@ -0,0 +1,95 @@
+#include "lib.h"
+#include "buffer.h"
+#include "hash-method.h"
+#include "hmac.h"
+#include "pkcs5.h"
+
+#include <stdint.h>
+#include <arpa/inet.h>
+
+static
+int pkcs5_pbkdf1(const struct hash_method *hash,
+	const unsigned char *password, size_t password_len,
+	const unsigned char *salt, size_t salt_len,
+	unsigned int iter, uint32_t length,
+	buffer_t *result)
+{
+	if (length < 1 ||
+		length > hash->digest_size) return -1;
+	if (iter < 1) return -1;
+
+	unsigned char dk[hash->digest_size];
+	unsigned char ctx[hash->context_size];
+
+	hash->init(ctx);
+	hash->loop(ctx, password, password_len);
+	hash->loop(ctx, salt, salt_len);
+	hash->result(ctx, dk);
+	length--;
+
+	for(;length>0;length--) {
+		hash->init(ctx);
+		hash->loop(ctx, dk, hash->digest_size);
+		hash->result(ctx, dk);
+	}
+
+	buffer_append(result, dk, hash->digest_size);
+
+	return 0;
+}
+
+static
+int pkcs5_pbkdf2(const struct hash_method *hash,
+	const unsigned char *password, size_t password_len,
+	const unsigned char *salt, size_t salt_len,
+	unsigned int iter, uint32_t length,
+	buffer_t *result)
+{
+	if (length < 1 || iter < 1) return -1;
+
+	size_t l = (length + hash->digest_size - 1)/hash->digest_size; /* same as ceil(length/hash->digest_size) */
+	unsigned char dk[l * hash->digest_size];
+	unsigned char *block;
+	struct hmac_context hctx;
+	unsigned int c,i,t;
+	unsigned char U_c[hash->digest_size];
+
+	for(t = 0; t < l; t++) {
+		block = &(dk[t*hash->digest_size]);
+		/* U_1 = PRF(Password, Salt|| INT_BE32(Block_Number)) */
+		c = htonl(t+1);
+		hmac_init(&hctx, password, password_len, hash);
+		hmac_update(&hctx, salt, salt_len);
+		hmac_update(&hctx, &c, sizeof(c));
+		hmac_final(&hctx, U_c);
+		/* block = U_1 ^ .. ^ U_iter */
+		memcpy(block, U_c, hash->digest_size);
+		/* U_c = PRF(Password, U_c-1) */
+		for(c = 1; c < iter; c++) {
+			hmac_init(&hctx, password, password_len, hash);
+			hmac_update(&hctx, U_c, hash->digest_size);
+			hmac_final(&hctx, U_c);
+			for(i = 0; i < hash->digest_size; i++)
+				block[i] ^= U_c[i];
+		}
+	}
+
+	buffer_append(result, dk, length);
+
+	return 0;
+}
+
+int pkcs5_pbkdf(enum pkcs5_pbkdf_mode mode, const struct hash_method *hash,
+	const unsigned char *password, size_t password_len,
+	const unsigned char *salt, size_t salt_len,
+	unsigned int iterations, uint32_t dk_len,
+	buffer_t *result)
+{
+	if (mode == PKCS5_PBKDF1)
+		return pkcs5_pbkdf1(hash,password,password_len,
+			salt,salt_len,iterations,dk_len,result);
+	else if (mode == PKCS5_PBKDF2)
+		return pkcs5_pbkdf2(hash,password,password_len,
+			salt,salt_len,iterations,dk_len,result);
+	i_unreached();
+}
diff --git a/src/lib/pkcs5.h b/src/lib/pkcs5.h
new file mode 100644
index 00000000000..4a249d48da4
--- /dev/null
+++ b/src/lib/pkcs5.h
@@ -0,0 +1,35 @@
+#ifndef PKCS5_H
+#define PKCS5_H 1
+
+enum pkcs5_pbkdf_mode {
+	PKCS5_PBKDF1,
+	PKCS5_PBKDF2
+};
+
+/*
+
+ mode         - v1.0 or v2.0
+ hash         - hash_method_lookup return value
+ password     - private password for generation
+ password_len - length of password in octets
+ salt         - salt for generation
+ salt_len     - length of salt in octets
+ iterations   - number of iterations to hash (use at least 1000, a very large number => very very slow)
+ dk_len       - number of bytes to return from derived key
+ result       - buffer_t to hold the result, either use dynamic or make sure it fits dk_len
+
+ non-zero return value indicates that either iterations was less than 1 or dk_len was too large
+
+ Sample code:
+
+ buffer_t *result = buffer_create_dynamic(pool_datastack_create(), 256);
+ if (pkcs5_pbkdf(PKCS5_PBKDF2, hash_method_lookup("sha256"), "password", 8, "salt", 4, 4096, 256, result) != 0) { // error }
+
+*/
+
+int pkcs5_pbkdf(enum pkcs5_pbkdf_mode mode, const struct hash_method *hash,
+	const unsigned char *password, size_t password_len,
+	const unsigned char *salt, size_t salt_len,
+	unsigned int iterations, uint32_t dk_len,
+	buffer_t *result);
+#endif
diff --git a/src/lib/test-lib.c b/src/lib/test-lib.c
index 325a37372c8..46f6f72fda9 100644
--- a/src/lib/test-lib.c
+++ b/src/lib/test-lib.c
@@ -39,6 +39,7 @@ int main(void)
 		test_mempool_alloconly,
 		test_net,
 		test_numpack,
+		test_pkcs5_pbkdf2,
 		test_ostream_escaped,
 		test_ostream_failure_at,
 		test_ostream_file,
diff --git a/src/lib/test-lib.h b/src/lib/test-lib.h
index e9cfe45ce08..23fc1eee6cf 100644
--- a/src/lib/test-lib.h
+++ b/src/lib/test-lib.h
@@ -39,6 +39,7 @@ void test_json_tree(void);
 void test_llist(void);
 void test_mempool_alloconly(void);
 enum fatal_test_state fatal_mempool(int);
+void test_pkcs5_pbkdf2(void);
 void test_net(void);
 void test_numpack(void);
 void test_ostream_escaped(void);
diff --git a/src/lib/test-pkcs5.c b/src/lib/test-pkcs5.c
new file mode 100644
index 00000000000..97b23b82501
--- /dev/null
+++ b/src/lib/test-pkcs5.c
@@ -0,0 +1,49 @@
+/* Copyright (c) 2007-2016 Dovecot authors, see the included COPYING file */
+
+#include "test-lib.h"
+#include "str.h"
+#include "buffer.h"
+#include "hash-method.h"
+#include "pkcs5.h"
+
+struct test_vector {
+	const char *prf;
+	unsigned char *p;
+	size_t pLen;
+	unsigned char *s;
+	size_t sLen;
+	unsigned int i;
+	unsigned char *dk;
+	size_t dkLen;
+};
+
+#define TEST_BUF(x) (unsigned char*)x, sizeof(x)-1
+
+/* RFC 6070 test vectors */
+static const struct test_vector test_vectors_v2[] = {
+	{ "sha1", TEST_BUF("password"), TEST_BUF("salt"), 1, TEST_BUF("\x0c\x60\xc8\x0f\x96\x1f\x0e\x71\xf3\xa9\xb5\x24\xaf\x60\x12\x06\x2f\xe0\x37\xa6") },
+	{ "sha1", TEST_BUF("password"), TEST_BUF("salt"), 2, TEST_BUF("\xea\x6c\x01\x4d\xc7\x2d\x6f\x8c\xcd\x1e\xd9\x2a\xce\x1d\x41\xf0\xd8\xde\x89\x57") },
+	{ "sha1", TEST_BUF("password"), TEST_BUF("salt"), 4096, TEST_BUF("\x4b\x00\x79\x01\xb7\x65\x48\x9a\xbe\xad\x49\xd9\x26\xf7\x21\xd0\x65\xa4\x29\xc1") },
+/* enable the next test only when you need it, it takes quite long time */
+/*	{ "sha1", TEST_BUF("password"), TEST_BUF("salt"), 16777216, TEST_BUF("\xee\xfe\x3d\x61\xcd\x4d\xa4\xe4\xe9\x94\x5b\x3d\x6b\xa2\x15\x8c\x26\x34\xe9\x84") }, */
+	{ "sha1", TEST_BUF("passwordPASSWORDpassword"), TEST_BUF("saltSALTsaltSALTsaltSALTsaltSALTsalt"), 4096, TEST_BUF("\x3d\x2e\xec\x4f\xe4\x1c\x84\x9b\x80\xc8\xd8\x36\x62\xc0\xe4\x4a\x8b\x29\x1a\x96\x4c\xf2\xf0\x70\x38") },
+	{ "sha1", TEST_BUF("pass\0word"), TEST_BUF("sa\0lt"), 4096, TEST_BUF("\x56\xfa\x6a\xa7\x55\x48\x09\x9d\xcc\x37\xd7\xf0\x34\x25\xe0\xc3") }
+};
+
+void test_pkcs5_pbkdf2(void)
+{
+	buffer_t *res = buffer_create_dynamic(default_pool, 25);
+
+	test_begin("pkcs5_pbkdf2");
+
+	for(size_t i = 0; i < N_ELEMENTS(test_vectors_v2); i++) {
+		buffer_reset(res);
+		const struct test_vector *vec = &(test_vectors_v2[i]);
+		pkcs5_pbkdf(PKCS5_PBKDF2, hash_method_lookup(vec->prf), vec->p, vec->pLen, vec->s, vec->sLen, vec->i, vec->dkLen, res);
+		test_assert_idx(memcmp(res->data, vec->dk, vec->dkLen) == 0, i);
+	}
+
+	buffer_free(&res);
+
+	test_end();
+}

From a5658a1143c4cf603aed81d6649448256a57a3eb Mon Sep 17 00:00:00 2001
From: Teemu Huovila <teemu.huovila@dovecot.fi>
Date: Fri, 29 Apr 2016 21:36:31 +0300
Subject: [PATCH 225/450] lib-dict-extra: Avoid out of scope data stack usage.

---
 src/lib-dict-extra/dict-ldap.c | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/lib-dict-extra/dict-ldap.c b/src/lib-dict-extra/dict-ldap.c
index d199ac8e1ed..770e562e5f8 100644
--- a/src/lib-dict-extra/dict-ldap.c
+++ b/src/lib-dict-extra/dict-ldap.c
@@ -162,13 +162,14 @@ int dict_ldap_connect(struct ldap_dict *dict, const char **error_r)
 	return ldap_client_init(&set, &dict->client, error_r);
 }
 
-static
-const char* ldap_dict_build_query(struct ldap_dict *dict, const struct dict_ldap_map *map, ARRAY_TYPE(const_string) *values, bool priv)
+static void
+ldap_dict_build_query(struct ldap_dict *dict, const struct dict_ldap_map *map,
+                      ARRAY_TYPE(const_string) *values, bool priv,
+                      string_t *query_r)
 {
 	const char *template;
 	ARRAY(struct var_expand_table) exp;
 	struct var_expand_table entry;
-	string_t *query = t_str_new(64);
 
 	t_array_init(&exp, 8);
 	entry.key = '\0';
@@ -191,9 +192,7 @@ const char* ldap_dict_build_query(struct ldap_dict *dict, const struct dict_ldap
 
 	array_append_zero(&exp);
 
-	var_expand(query, template, array_idx(&exp, 0));
-
-	return str_c(query);
+	var_expand(query_r, template, array_idx(&exp, 0));
 }
 
 static
@@ -377,6 +376,7 @@ void ldap_dict_lookup_async(struct dict *dict, const char *key,
 	struct ldap_dict *ctx = (struct ldap_dict*)dict;
 	struct dict_ldap_op *op;
 	pool_t oppool = pool_alloconly_create("ldap dict lookup", 64);
+	string_t *query = str_new(oppool, 64);
 	op = p_new(oppool, struct dict_ldap_op, 1);
 	op->pool = oppool;
 	op->dict = ctx;
@@ -398,7 +398,8 @@ void ldap_dict_lookup_async(struct dict *dict, const char *key,
 			memset(&input, 0, sizeof(input));
 			input.base_dn = map->base_dn;
 			input.scope = map->scope_val;
-			input.filter = ldap_dict_build_query(ctx, map, &values, strncmp(key, DICT_PATH_PRIVATE, strlen(DICT_PATH_PRIVATE))==0);
+			ldap_dict_build_query(ctx, map, &values, strncmp(key, DICT_PATH_PRIVATE, strlen(DICT_PATH_PRIVATE))==0, query);
+			input.filter = str_c(query);
 			input.attributes = attributes;
 			input.timeout_secs = ctx->set->timeout;
 			ctx->pending++;

From 0fbf2c9348b4875d331595100ed40a121e40c23d Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Tue, 26 Apr 2016 19:26:34 +0300
Subject: [PATCH 226/450] doveadm-server: Handle istream send error.

---
 src/doveadm/client-connection-http.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/doveadm/client-connection-http.c b/src/doveadm/client-connection-http.c
index 7fa530ba494..30c6d19610e 100644
--- a/src/doveadm/client-connection-http.c
+++ b/src/doveadm/client-connection-http.c
@@ -189,7 +189,13 @@ static void doveadm_http_server_json_success(void *context, struct istream *resu
 	string_t *escaped;
 	escaped = str_new(conn->client.pool, 10);
 	o_stream_nsend_str(conn->client.output,"[\"doveadmResponse\",");
-	o_stream_send_istream(conn->client.output, result);
+	if (o_stream_send_istream(conn->client.output, result) < 0) {
+		if (conn->client.output->stream_errno != 0) {
+			i_fatal("write(%s) failed: %s", o_stream_get_name(conn->client.output), o_stream_get_error(conn->client.output));
+		} else if (result->stream_errno != 0) {
+			i_fatal("read(%s) failed: %s", i_stream_get_name(result), i_stream_get_error(result));
+		} else i_unreached(); /* either it's output or input error */
+	}
 	o_stream_nsend_str(conn->client.output,",\"");
 	if (conn->method_id != NULL) {
 		json_append_escaped(escaped, conn->method_id);

From 5c33861d5d0e6d00e03446bca6501c7ace1f5ae7 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sat, 30 Apr 2016 15:26:27 +0300
Subject: [PATCH 227/450] .gitignore: Added *~

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index 065f4118d5b..a4343946b89 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,6 +8,7 @@ Makefile.in
 *.rej
 .deps
 .libs
+*~
 
 /configure
 /config.cache

From 273ed7a132cff5ae191a195d297c626f30e556b4 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@rename-it.nl>
Date: Sat, 30 Apr 2016 14:22:21 +0200
Subject: [PATCH 228/450] lib-http: server: Fixed assert failure occurring when
 closing the connection while a request payload was still being read.

---
 src/lib-http/http-server-connection.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/lib-http/http-server-connection.c b/src/lib-http/http-server-connection.c
index 94713e6805b..be3d39055e0 100644
--- a/src/lib-http/http-server-connection.c
+++ b/src/lib-http/http-server-connection.c
@@ -1062,6 +1062,13 @@ http_server_connection_disconnect(struct http_server_connection *conn,
 	/* preserve statistics */
 	http_server_connection_update_stats(conn);
 
+	if (conn->incoming_payload != NULL) {
+		/* the stream is still accessed by lib-http caller. */
+		i_stream_remove_destroy_callback(conn->incoming_payload,
+						 http_server_payload_destroyed);
+		conn->incoming_payload = NULL;
+	}
+
 	/* drop all requests before connection is closed */
 	req = conn->request_queue_head;
 	while (req != NULL) {
@@ -1081,13 +1088,6 @@ http_server_connection_disconnect(struct http_server_connection *conn,
 		o_stream_uncork(conn->conn.output);
 	}
 
-	if (conn->incoming_payload != NULL) {
-		/* the stream is still accessed by lib-http caller. */
-		i_stream_remove_destroy_callback(conn->incoming_payload,
-						 http_server_payload_destroyed);
-		conn->incoming_payload = NULL;
-	}
-
 	if (conn->http_parser != NULL)
 		http_request_parser_deinit(&conn->http_parser);
 	connection_disconnect(&conn->conn);

From 2812b0ea5d60e092ed3228589cba74de22fa74bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martti=20Rannanj=C3=A4rvi?= <martti.rannanjarvi@dovecot.fi>
Date: Wed, 20 Apr 2016 14:33:57 +0300
Subject: [PATCH 229/450] doveadm: error to print formatted without format

This changes the segfault of `doveadm -f formatted ...` to an error.
---
 src/doveadm/doveadm-print-formatted.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/doveadm/doveadm-print-formatted.c b/src/doveadm/doveadm-print-formatted.c
index 87e03c5f199..28347eb81d4 100644
--- a/src/doveadm/doveadm-print-formatted.c
+++ b/src/doveadm/doveadm-print-formatted.c
@@ -53,6 +53,9 @@ static void doveadm_print_formatted_flush(void)
 
 static void doveadm_print_formatted_print(const char *value)
 {
+	if (ctx.format == NULL) {
+		i_fatal("formatted formatter cannot be used without a format.");
+	}
 	struct var_expand_table *entry = array_idx_modifiable(&ctx.headers, ctx.idx++);
 	entry->value = value;
 

From 7ec76ff542059eb25bb1cc6f8aafa3b9e093b908 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 3 May 2016 14:54:28 +0300
Subject: [PATCH 230/450] doveadm fetch: Added body.<section> and
 binary.<section> fields

These correspond to IMAP BODY[<section>].
---
 src/doveadm/Makefile.am          |  1 +
 src/doveadm/doveadm-mail-fetch.c | 47 ++++++++++++++++++++++++++++++--
 2 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/src/doveadm/Makefile.am b/src/doveadm/Makefile.am
index cfdaf2152c2..785dc4010a6 100644
--- a/src/doveadm/Makefile.am
+++ b/src/doveadm/Makefile.am
@@ -20,6 +20,7 @@ AM_CPPFLAGS = \
 	-I$(top_srcdir)/src/lib-imap \
 	-I$(top_srcdir)/src/lib-index \
 	-I$(top_srcdir)/src/lib-storage \
+	-I$(top_srcdir)/src/lib-imap-storage \
 	-I$(top_srcdir)/src/lib-http \
 	-I$(top_srcdir)/src/auth \
 	-DMODULEDIR=\""$(moduledir)"\" \
diff --git a/src/doveadm/doveadm-mail-fetch.c b/src/doveadm/doveadm-mail-fetch.c
index d46aaa6910e..c66220fe63e 100644
--- a/src/doveadm/doveadm-mail-fetch.c
+++ b/src/doveadm/doveadm-mail-fetch.c
@@ -15,6 +15,7 @@
 #include "mail-storage.h"
 #include "mail-search.h"
 #include "mail-namespace.h"
+#include "imap-msgpart.h"
 #include "doveadm-print.h"
 #include "doveadm-mail.h"
 #include "doveadm-mailbox-list-iter.h"
@@ -186,6 +187,31 @@ static int fetch_hdr_field(struct fetch_cmd_context *ctx)
 	return 0;
 }
 
+static int fetch_body_field(struct fetch_cmd_context *ctx)
+{
+	const char *name = ctx->cur_field->name;
+	struct imap_msgpart *msgpart;
+	struct imap_msgpart_open_result result;
+	bool binary;
+	int ret;
+
+	binary = strncmp(name, "binary.", 7) == 0;
+	name += binary ? 7 : 5;
+	if (imap_msgpart_parse(name, &msgpart) < 0)
+		i_unreached(); /* we already verified this was ok */
+	if (binary)
+		imap_msgpart_set_decode_to_binary(msgpart);
+
+	if (imap_msgpart_open(ctx->mail, msgpart, &result) < 0) {
+		imap_msgpart_free(&msgpart);
+		return -1;
+	}
+	ret = doveadm_print_istream(result.input);
+	i_stream_unref(&result.input);
+	imap_msgpart_free(&msgpart);
+	return ret;
+}
+
 static int fetch_body(struct fetch_cmd_context *ctx)
 {
 	struct istream *input;
@@ -476,7 +502,7 @@ static void print_fetch_fields(void)
 {
 	unsigned int i;
 
-	fprintf(stderr, "Available fetch fields: %s", fetch_fields[0].name);
+	fprintf(stderr, "Available fetch fields: hdr.<name> body.<section> binary.<section> %s", fetch_fields[0].name);
 	for (i = 1; i < N_ELEMENTS(fetch_fields); i++)
 		fprintf(stderr, " %s", fetch_fields[i].name);
 	fprintf(stderr, "\n");
@@ -486,11 +512,15 @@ static void parse_fetch_fields(struct fetch_cmd_context *ctx, const char *str)
 {
 	const char *const *fields, *name;
 	const struct fetch_field *field;
-	struct fetch_field hdr_field;
+	struct fetch_field hdr_field, body_field;
+	struct imap_msgpart *msgpart;
 
 	memset(&hdr_field, 0, sizeof(hdr_field));
 	hdr_field.print = fetch_hdr_field;
 
+	memset(&body_field, 0, sizeof(body_field));
+	body_field.print = fetch_body_field;
+
 	t_array_init(&ctx->fields, 32);
 	t_array_init(&ctx->header_fields, 32);
 	fields = t_strsplit_spaces(str, " ");
@@ -504,6 +534,19 @@ static void parse_fetch_fields(struct fetch_cmd_context *ctx, const char *str)
 			array_append(&ctx->fields, &hdr_field, 1);
 			name = t_strcut(name, '.');
 			array_append(&ctx->header_fields, &name, 1);
+		} else if (strncmp(name, "body.", 5) == 0 ||
+			   strncmp(name, "binary.", 7) == 0) {
+			bool binary = strncmp(name, "binary.", 7) == 0;
+			body_field.name = name;
+
+			name += binary ? 7 : 5;
+			if (imap_msgpart_parse(name, &msgpart) < 0) {
+				print_fetch_fields();
+				i_fatal("Unknown fetch section: %s", name);
+			}
+			array_append(&ctx->fields, &body_field, 1);
+			ctx->wanted_fields |= imap_msgpart_get_fetch_data(msgpart);
+			imap_msgpart_free(&msgpart);
 		} else {
 			field = fetch_field_find(name);
 			if (field == NULL) {

From 334702e6d61d2c767d56b539a7275a8e4bf3c20b Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 3 May 2016 14:58:37 +0300
Subject: [PATCH 231/450] doveadm fetch: Fixed logging errors for istream
 failures

---
 src/doveadm/doveadm-mail-fetch.c | 31 ++++++++++++++++++++++++++-----
 1 file changed, 26 insertions(+), 5 deletions(-)

diff --git a/src/doveadm/doveadm-mail-fetch.c b/src/doveadm/doveadm-mail-fetch.c
index c66220fe63e..ab500544685 100644
--- a/src/doveadm/doveadm-mail-fetch.c
+++ b/src/doveadm/doveadm-mail-fetch.c
@@ -33,6 +33,9 @@ struct fetch_cmd_context {
 	enum mail_fetch_field wanted_fields;
 
 	const struct fetch_field *cur_field;
+	/* if print() returns -1, log this error if non-NULL. otherwise log
+	   the storage error. */
+	const char *print_error;
 };
 
 struct fetch_field {
@@ -107,6 +110,13 @@ static int fetch_modseq(struct fetch_cmd_context *ctx)
 	return 0;
 }
 
+static void
+fetch_set_istream_error(struct fetch_cmd_context *ctx, struct istream *input)
+{
+	ctx->print_error = t_strdup_printf("read(%s) failed: %s",
+		i_stream_get_name(input), i_stream_get_error(input));
+}
+
 static int fetch_hdr(struct fetch_cmd_context *ctx)
 {
 	struct istream *input;
@@ -117,7 +127,8 @@ static int fetch_hdr(struct fetch_cmd_context *ctx)
 		return -1;
 
 	input = i_stream_create_limit(input, hdr_size.physical_size);
-	ret = doveadm_print_istream(input);
+	if ((ret = doveadm_print_istream(input) < 0))
+		fetch_set_istream_error(ctx, input);
 	i_stream_unref(&input);
 	return ret;
 }
@@ -206,7 +217,8 @@ static int fetch_body_field(struct fetch_cmd_context *ctx)
 		imap_msgpart_free(&msgpart);
 		return -1;
 	}
-	ret = doveadm_print_istream(result.input);
+	if ((ret = doveadm_print_istream(result.input) < 0))
+		fetch_set_istream_error(ctx, result.input);
 	i_stream_unref(&result.input);
 	imap_msgpart_free(&msgpart);
 	return ret;
@@ -216,12 +228,15 @@ static int fetch_body(struct fetch_cmd_context *ctx)
 {
 	struct istream *input;
 	struct message_size hdr_size;
+	int ret;
 
 	if (mail_get_stream(ctx->mail, &hdr_size, NULL, &input) < 0)
 		return -1;
 
 	i_stream_skip(input, hdr_size.physical_size);
-	return doveadm_print_istream(input);
+	if ((ret = doveadm_print_istream(input) < 0))
+		fetch_set_istream_error(ctx, input);
+	return ret;
 }
 
 static int fetch_body_snippet(struct fetch_cmd_context *ctx)
@@ -239,10 +254,13 @@ static int fetch_body_snippet(struct fetch_cmd_context *ctx)
 static int fetch_text(struct fetch_cmd_context *ctx)
 {
 	struct istream *input;
+	int ret;
 
 	if (mail_get_stream(ctx->mail, NULL, NULL, &input) < 0)
 		return -1;
-	return doveadm_print_istream(input);
+	if ((ret = doveadm_print_istream(input) < 0))
+		fetch_set_istream_error(ctx, input);
+	return ret;
 }
 
 static int fetch_text_utf8(struct fetch_cmd_context *ctx)
@@ -571,8 +589,11 @@ static int cmd_fetch_mail(struct fetch_cmd_context *ctx)
 		if (field->print(ctx) < 0) {
 			i_error("fetch(%s) failed for box=%s uid=%u: %s",
 				field->name, mailbox_get_vname(mail->box),
-				mail->uid, mailbox_get_last_error(mail->box, NULL));
+				mail->uid,
+				ctx->print_error != NULL ? ctx->print_error :
+				mailbox_get_last_error(mail->box, NULL));
 			doveadm_mail_failed_mailbox(&ctx->ctx, mail->box);
+			ctx->print_error = NULL;
 			ret = -1;
 		}
 	}

From 48713ea8a05c8be974e627ce82bc52d72e723db9 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 3 May 2016 19:39:44 +0300
Subject: [PATCH 232/450] lib-imap-storage: Allow HEADER.FIELDS(..) without
 space before '('

This is to allow doveadm fetch to use it. Attempting to do this without a
space in IMAP already fails elsewhere:

x fetch 1 body.peek[header.fields(from to)]
x BAD Error in IMAP command FETCH: Invalid characters in atom
---
 src/lib-imap-storage/imap-msgpart.c | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/lib-imap-storage/imap-msgpart.c b/src/lib-imap-storage/imap-msgpart.c
index d8039e0caa6..37e85b9ee9a 100644
--- a/src/lib-imap-storage/imap-msgpart.c
+++ b/src/lib-imap-storage/imap-msgpart.c
@@ -184,6 +184,9 @@ imap_msgpart_parse_header_fields(struct imap_msgpart *msgpart,
 {
 	ARRAY_TYPE(const_string) fields;
 
+	if (header_list[0] == ' ')
+		header_list++;
+
 	/* HEADER.FIELDS (list), HEADER.FIELDS.NOT (list) */
 	if (imap_msgpart_get_header_fields(msgpart->pool, header_list,
 					   &fields) < 0)
@@ -267,14 +270,14 @@ int imap_msgpart_parse(const char *section, struct imap_msgpart **msgpart_r)
 		if (section[6] == '\0') {
 			msgpart->fetch_type = FETCH_HEADER;
 			ret = 0;
-		} else if (strncmp(section, "HEADER.FIELDS ", 14) == 0) {
-			msgpart->fetch_type = FETCH_HEADER_FIELDS;
-			ret = imap_msgpart_parse_header_fields(msgpart,
-							       section+14);
-		} else if (strncmp(section, "HEADER.FIELDS.NOT ", 18) == 0) {
+		} else if (strncmp(section, "HEADER.FIELDS.NOT", 17) == 0) {
 			msgpart->fetch_type = FETCH_HEADER_FIELDS_NOT;
 			ret = imap_msgpart_parse_header_fields(msgpart,
-							       section+18);
+							       section+17);
+		} else if (strncmp(section, "HEADER.FIELDS", 13) == 0) {
+			msgpart->fetch_type = FETCH_HEADER_FIELDS;
+			ret = imap_msgpart_parse_header_fields(msgpart,
+							       section+13);
 		} else {
 			ret = -1;
 		}

From 3def3a7f6fdf56d0fb27149a769e73dd172dadb2 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 3 May 2016 19:42:08 +0300
Subject: [PATCH 233/450] doveadm fetch: Translate commas to spaces in
 body|binary.<section> with

This way we can do e.g.:

doveadm fetch 'uid body.header.fields(from,to,subject)' ...

Although in theory we could also change the parser to allow spaces as long
as they're inside (..), but this is probably too much extra effort.
---
 src/doveadm/doveadm-mail-fetch.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/doveadm/doveadm-mail-fetch.c b/src/doveadm/doveadm-mail-fetch.c
index ab500544685..2fd58976bdb 100644
--- a/src/doveadm/doveadm-mail-fetch.c
+++ b/src/doveadm/doveadm-mail-fetch.c
@@ -555,7 +555,7 @@ static void parse_fetch_fields(struct fetch_cmd_context *ctx, const char *str)
 		} else if (strncmp(name, "body.", 5) == 0 ||
 			   strncmp(name, "binary.", 7) == 0) {
 			bool binary = strncmp(name, "binary.", 7) == 0;
-			body_field.name = name;
+			body_field.name = t_strarray_join(t_strsplit(name, ","), " ");
 
 			name += binary ? 7 : 5;
 			if (imap_msgpart_parse(name, &msgpart) < 0) {

From 804a5518b27282d847dc04e228024a36fe9fac9d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 11 May 2016 05:19:14 -0400
Subject: [PATCH 234/450] dsync: Fixed potential crash

This seems to be possible with dsync_mailbox_find_common_expunged_uid() ->
dsync_mailbox_common_uid_found() -> dsync_mailbox_rewind_search() -> following
search doesn't find anything.
---
 src/doveadm/dsync/dsync-mailbox-import.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c
index 6b5c6bf7554..10dc3571f2b 100644
--- a/src/doveadm/dsync/dsync-mailbox-import.c
+++ b/src/doveadm/dsync/dsync-mailbox-import.c
@@ -1688,7 +1688,7 @@ dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,
 		(void)dsync_mailbox_find_common_expunged_uid(importer, change, result_r);
 	}
 	*result_r = t_strdup_printf("%s (next local mail UID=%u)",
-				    *result_r, importer->cur_mail->uid);
+		*result_r, importer->cur_mail == NULL ? 0 : importer->cur_mail->uid);
 }
 
 int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,

From b1254dfe442639236f881afae85e0739520ab409 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@rename-it.nl>
Date: Tue, 10 May 2016 20:37:55 +0200
Subject: [PATCH 235/450] Added LIBDOVECOT_SQL_INCLUDE to dovecot-config and
 dovecot.m4.

---
 dovecot-config.in.in | 1 +
 dovecot.m4           | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/dovecot-config.in.in b/dovecot-config.in.in
index c71bf52d5d2..8328de4d340 100644
--- a/dovecot-config.in.in
+++ b/dovecot-config.in.in
@@ -29,6 +29,7 @@ LIBDOVECOT_DOVEADM_INCLUDE="-I$(incdir)/src/doveadm"
 LIBDOVECOT_STORAGE_INCLUDE="-I$(incdir)/src/lib-index -I$(incdir)/src/lib-storage -I$(incdir)/src/lib-storage/list -I$(incdir)/src/lib-storage/index -I$(incdir)/src/lib-storage/index/raw -I$(incdir)/src/lib-imap-storage -I$(incdir)/src/plugins/quota"
 LIBDOVECOT_DSYNC_INCLUDE="-I$(incdir)/src/doveadm/dsync"
 LIBDOVECOT_LOGIN_INCLUDE="-I$(incdir)/src/login-common"
+LIBDOVECOT_SQL_INCLUDE="-I$(incdir)/src/lib-sql"
 LIBDOVECOT_IMAP_LOGIN_INCLUDE="-I$(incdir)/src/imap-login"
 LIBDOVECOT_IMAP_INCLUDE="-I$(incdir)/src/imap"
 LIBDOVECOT_POP3_INCLUDE="-I$(incdir)/src/pop3"
diff --git a/dovecot.m4 b/dovecot.m4
index 30b9ae258fb..efecf1c3bb3 100644
--- a/dovecot.m4
+++ b/dovecot.m4
@@ -6,7 +6,7 @@
 # unlimited permission to copy and/or distribute it, with or without
 # modifications, as long as this notice is preserved.
 
-# serial 21
+# serial 22
 
 AC_DEFUN([DC_DOVECOT_MODULEDIR],[
 	AC_ARG_WITH(moduledir,
@@ -128,7 +128,7 @@ AC_DEFUN([DC_DOVECOT],[
 	AX_SUBST_L([DOVECOT_INSTALLED], [DOVECOT_CFLAGS], [DOVECOT_LIBS], [DOVECOT_SSL_LIBS], [DOVECOT_SQL_LIBS], [DOVECOT_COMPRESS_LIBS])
 	AX_SUBST_L([LIBDOVECOT], [LIBDOVECOT_LOGIN], [LIBDOVECOT_SQL], [LIBDOVECOT_SSL], [LIBDOVECOT_COMPRESS], [LIBDOVECOT_LDA], [LIBDOVECOT_STORAGE], [LIBDOVECOT_DSYNC], [LIBDOVECOT_LIBFTS])
 	AX_SUBST_L([LIBDOVECOT_DEPS], [LIBDOVECOT_LOGIN_DEPS], [LIBDOVECOT_SQL_DEPS], [LIBDOVECOT_SSL_DEPS], [LIBDOVECOT_COMPRESS_DEPS], [LIBDOVECOT_LDA_DEPS], [LIBDOVECOT_STORAGE_DEPS], [LIBDOVECOT_DSYNC_DEPS], [LIBDOVECOT_LIBFTS_DEPS])
-	AX_SUBST_L([LIBDOVECOT_INCLUDE], [LIBDOVECOT_LDA_INCLUDE], [LIBDOVECOT_AUTH_INCLUDE], [LIBDOVECOT_DOVEADM_INCLUDE], [LIBDOVECOT_SERVICE_INCLUDE], [LIBDOVECOT_STORAGE_INCLUDE], [LIBDOVECOT_LOGIN_INCLUDE], [LIBDOVECOT_IMAP_LOGIN_INCLUDE], [LIBDOVECOT_CONFIG_INCLUDE], [LIBDOVECOT_IMAP_INCLUDE], [LIBDOVECOT_POP3_INCLUDE], [LIBDOVECOT_DSYNC_INCLUDE], [LIBDOVECOT_IMAPC_INCLUDE], [LIBDOVECOT_FTS_INCLUDE], [LIBDOVECOT_NOTIFY_INCLUDE], [LIBDOVECOT_ACL_INCLUDE], [LIBDOVECOT_LIBFTS_INCLUDE])
+	AX_SUBST_L([LIBDOVECOT_INCLUDE], [LIBDOVECOT_LDA_INCLUDE], [LIBDOVECOT_AUTH_INCLUDE], [LIBDOVECOT_DOVEADM_INCLUDE], [LIBDOVECOT_SERVICE_INCLUDE], [LIBDOVECOT_STORAGE_INCLUDE], [LIBDOVECOT_LOGIN_INCLUDE], [LIBDOVECOT_SQL_INCLUDE], [LIBDOVECOT_IMAP_LOGIN_INCLUDE], [LIBDOVECOT_CONFIG_INCLUDE], [LIBDOVECOT_IMAP_INCLUDE], [LIBDOVECOT_POP3_INCLUDE], [LIBDOVECOT_DSYNC_INCLUDE], [LIBDOVECOT_IMAPC_INCLUDE], [LIBDOVECOT_FTS_INCLUDE], [LIBDOVECOT_NOTIFY_INCLUDE], [LIBDOVECOT_ACL_INCLUDE], [LIBDOVECOT_LIBFTS_INCLUDE])
 
 	AM_CONDITIONAL(DOVECOT_INSTALLED, test "$DOVECOT_INSTALLED" = "yes")
 

From 44069b06572637549f51a19e24c5c2d3404010d8 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 16 May 2016 15:49:08 +0300
Subject: [PATCH 236/450] global: Use [io]_stream_get_error() insted of %m

---
 src/auth/mech-winbind.c                       |  3 ++-
 src/auth/userdb-passwd-file.c                 |  3 ++-
 src/director/director-connection.c            |  7 +++---
 src/doveadm/doveadm-director.c                | 15 ++++++-----
 src/doveadm/doveadm-penalty.c                 |  6 +++--
 src/doveadm/doveadm-replicator.c              | 13 ++++++----
 src/doveadm/doveadm-stats.c                   |  8 +++---
 src/doveadm/doveadm-who.c                     |  6 +++--
 src/doveadm/doveadm-zlib.c                    |  4 +--
 src/lib-auth/auth-master.c                    |  3 ++-
 src/lib-auth/auth-server-connection.c         |  5 ++--
 src/lib-http/http-server-response.c           | 25 ++++++++++---------
 src/lib-index/mail-index-strmap.c             |  4 +--
 src/lib-lda/duplicate.c                       |  3 ++-
 src/lib-lda/lmtp-client.c                     |  4 +--
 src/lib-lda/smtp-client.c                     |  8 +++---
 src/lib-master/anvil-client.c                 |  3 ++-
 src/lib-master/master-instance.c              |  4 +--
 src/lib-master/master-service-settings.c      |  3 ++-
 src/lib-storage/index/cydir/cydir-save.c      |  3 ++-
 .../index/dbox-multi/mdbox-purge.c            | 20 +++++++--------
 .../index/dbox-single/sdbox-file.c            | 15 ++++-------
 src/lib-storage/index/imapc/imapc-save.c      |  3 ++-
 src/lib-storage/index/index-attachment.c      |  5 ++--
 src/lib-storage/index/maildir/maildir-save.c  | 13 +++++-----
 .../index/maildir/maildir-uidlist.c           |  7 +++---
 src/lib/json-parser.c                         |  5 ++--
 src/plugins/acl/acl-backend-vfile-acllist.c   |  3 ++-
 src/plugins/acl/acl-backend-vfile-update.c    |  3 ++-
 src/plugins/acl/acl-backend-vfile.c           |  3 ++-
 src/plugins/fts-squat/squat-trie.c            |  3 ++-
 src/plugins/fts-squat/squat-uidlist.c         |  6 +++--
 src/plugins/fts/fts-expunge-log.c             |  3 ++-
 33 files changed, 123 insertions(+), 96 deletions(-)

diff --git a/src/auth/mech-winbind.c b/src/auth/mech-winbind.c
index c9e1450f0ba..4a656961051 100644
--- a/src/auth/mech-winbind.c
+++ b/src/auth/mech-winbind.c
@@ -180,7 +180,8 @@ do_auth_continue(struct auth_request *auth_request,
 			  str_data(str), str_len(str)) < 0 ||
 	    o_stream_flush(request->winbind->out_pipe) < 0) {
 		auth_request_log_error(auth_request, AUTH_SUBSYS_MECH,
-				       "write(out_pipe) failed: %m");
+			"write(out_pipe) failed: %s",
+			o_stream_get_error(request->winbind->out_pipe));
 		return HR_RESTART;
 	}
 	request->continued = FALSE;
diff --git a/src/auth/userdb-passwd-file.c b/src/auth/userdb-passwd-file.c
index b578d272b6c..f80cac1f9e0 100644
--- a/src/auth/userdb-passwd-file.c
+++ b/src/auth/userdb-passwd-file.c
@@ -139,7 +139,8 @@ static void passwd_file_iterate_next(struct userdb_iterate_context *_ctx)
 			break;
 		}
 		if (line == NULL && ctx->input->stream_errno != 0) {
-			i_error("read(%s) failed: %m", ctx->path);
+			i_error("read(%s) failed: %s", ctx->path,
+				i_stream_get_error(ctx->input));
 			_ctx->failed = TRUE;
 		}
 	}
diff --git a/src/director/director-connection.c b/src/director/director-connection.c
index a727658efd7..2beb3270735 100644
--- a/src/director/director-connection.c
+++ b/src/director/director-connection.c
@@ -2051,9 +2051,10 @@ void director_connection_send(struct director_connection *conn,
 	} T_END;
 	ret = o_stream_send(conn->output, data, len);
 	if (ret != (off_t)len) {
-		if (ret < 0)
-			i_error("director(%s): write() failed: %m", conn->name);
-		else {
+		if (ret < 0) {
+			i_error("director(%s): write() failed: %s", conn->name,
+				o_stream_get_error(conn->output));
+		} else {
 			i_error("director(%s): Output buffer full, "
 				"disconnecting", conn->name);
 		}
diff --git a/src/doveadm/doveadm-director.c b/src/doveadm/doveadm-director.c
index 2d6de55d832..46b2cae2ee8 100644
--- a/src/doveadm/doveadm-director.c
+++ b/src/doveadm/doveadm-director.c
@@ -66,11 +66,12 @@ static void director_connect(struct director_context *ctx)
 	line = i_stream_read_next_line(ctx->input);
 	alarm(0);
 	if (line == NULL) {
-		if (ctx->input->stream_errno != 0)
-			i_fatal("read(%s) failed: %m", ctx->socket_path);
-		else if (ctx->input->eof)
+		if (ctx->input->stream_errno != 0) {
+			i_fatal("read(%s) failed: %s", ctx->socket_path,
+				i_stream_get_error(ctx->input));
+		} else if (ctx->input->eof) {
 			i_fatal("%s disconnected", ctx->socket_path);
-		else {
+		} else {
 			i_fatal("read(%s) timed out (is director configured?)",
 				ctx->socket_path);
 		}
@@ -85,8 +86,10 @@ static void director_connect(struct director_context *ctx)
 static void director_disconnect(struct director_context *ctx)
 {
 	if (ctx->input != NULL) {
-		if (ctx->input->stream_errno != 0)
-			i_fatal("read(%s) failed: %m", ctx->socket_path);
+		if (ctx->input->stream_errno != 0) {
+			i_fatal("read(%s) failed: %s", ctx->socket_path,
+				i_stream_get_error(ctx->input));
+		}
 		i_stream_destroy(&ctx->input);
 	}
 }
diff --git a/src/doveadm/doveadm-penalty.c b/src/doveadm/doveadm-penalty.c
index 36b9fd85f4b..2a4b33d8878 100644
--- a/src/doveadm/doveadm-penalty.c
+++ b/src/doveadm/doveadm-penalty.c
@@ -80,8 +80,10 @@ static void penalty_lookup(struct penalty_context *ctx)
 			penalty_print_line(ctx, &penalty_line);
 		} T_END;
 	}
-	if (input->stream_errno != 0)
-		i_fatal("read(%s) failed: %m", ctx->anvil_path);
+	if (input->stream_errno != 0) {
+		i_fatal("read(%s) failed: %s", ctx->anvil_path,
+			i_stream_get_error(input));
+	}
 
 	i_stream_destroy(&input);
 }
diff --git a/src/doveadm/doveadm-replicator.c b/src/doveadm/doveadm-replicator.c
index 6459a6c0f9f..390572b530f 100644
--- a/src/doveadm/doveadm-replicator.c
+++ b/src/doveadm/doveadm-replicator.c
@@ -48,9 +48,10 @@ static void replicator_connect(struct replicator_context *ctx)
 	line = i_stream_read_next_line(ctx->input);
 	alarm(0);
 	if (line == NULL) {
-		if (ctx->input->stream_errno != 0)
-			i_fatal("read(%s) failed: %m", ctx->socket_path);
-		else if (ctx->input->eof)
+		if (ctx->input->stream_errno != 0) {
+			i_fatal("read(%s) failed: %s", ctx->socket_path,
+				i_stream_get_error(ctx->input));
+		} else if (ctx->input->eof)
 			i_fatal("%s disconnected", ctx->socket_path);
 		else
 			i_fatal("read(%s) timed out", ctx->socket_path);
@@ -64,8 +65,10 @@ static void replicator_connect(struct replicator_context *ctx)
 
 static void replicator_disconnect(struct replicator_context *ctx)
 {
-	if (ctx->input->stream_errno != 0)
-		i_fatal("read(%s) failed: %m", ctx->socket_path);
+	if (ctx->input->stream_errno != 0) {
+		i_fatal("read(%s) failed: %s", ctx->socket_path,
+			i_stream_get_error(ctx->input));
+	}
 	i_stream_destroy(&ctx->input);
 }
 
diff --git a/src/doveadm/doveadm-stats.c b/src/doveadm/doveadm-stats.c
index 81e6f25623b..54201ed78a8 100644
--- a/src/doveadm/doveadm-stats.c
+++ b/src/doveadm/doveadm-stats.c
@@ -111,7 +111,7 @@ static void stats_dump(const char *path, const char *cmd)
 		} while (args != NULL);
 	}
 	if (input->stream_errno != 0)
-		i_fatal("read(%s) failed: %m", path);
+		i_fatal("read(%s) failed: %s", path, i_stream_get_error(input));
 	i_stream_destroy(&input);
 }
 
@@ -194,8 +194,10 @@ static void stats_read(struct top_context *ctx)
 		hash_table_insert(ctx->sessions, line->id, line);
 	}
 
-	if (ctx->input->stream_errno != 0)
-		i_fatal("read(%s) failed: %m", ctx->path);
+	if (ctx->input->stream_errno != 0) {
+		i_fatal("read(%s) failed: %s", ctx->path,
+			i_stream_get_error(ctx->input));
+	}
 	i_fatal("read(%s): unexpected EOF", ctx->path);
 }
 
diff --git a/src/doveadm/doveadm-who.c b/src/doveadm/doveadm-who.c
index 3b75b9a9a69..d0f3b0f5d73 100644
--- a/src/doveadm/doveadm-who.c
+++ b/src/doveadm/doveadm-who.c
@@ -169,8 +169,10 @@ void who_lookup(struct who_context *ctx, who_callback_t *callback)
 				callback(ctx, &who_line);
 		} T_END;
 	}
-	if (input->stream_errno != 0)
-		i_fatal("read(%s) failed: %m", ctx->anvil_path);
+	if (input->stream_errno != 0) {
+		i_fatal("read(%s) failed: %s", ctx->anvil_path,
+			i_stream_get_error(input));
+	}
 
 	i_stream_destroy(&input);
 }
diff --git a/src/doveadm/doveadm-zlib.c b/src/doveadm/doveadm-zlib.c
index a459a46058a..fe6f19be71f 100644
--- a/src/doveadm/doveadm-zlib.c
+++ b/src/doveadm/doveadm-zlib.c
@@ -122,8 +122,8 @@ static void server_input(struct client *client)
 
 	if (i_stream_read(client->input) == -1) {
 		if (client->input->stream_errno != 0) {
-			errno = client->input->stream_errno;
-			i_fatal("read(server) failed: %m");
+			i_fatal("read(server) failed: %s",
+				i_stream_get_error(client->input));
 		}
 
 		i_info("Server disconnected");
diff --git a/src/lib-auth/auth-master.c b/src/lib-auth/auth-master.c
index a138417e206..b3ceab510f8 100644
--- a/src/lib-auth/auth-master.c
+++ b/src/lib-auth/auth-master.c
@@ -413,7 +413,8 @@ static int auth_master_run_cmd_pre(struct auth_master_connection *conn,
 	o_stream_uncork(conn->output);
 
 	if (o_stream_nfinish(conn->output) < 0) {
-		i_error("write(auth socket) failed: %m");
+		i_error("write(auth socket) failed: %s",
+			o_stream_get_error(conn->output));
 		auth_master_unset_io(conn);
 		auth_connection_close(conn);
 		return -1;
diff --git a/src/lib-auth/auth-server-connection.c b/src/lib-auth/auth-server-connection.c
index 528d49a6714..8453a3a61f4 100644
--- a/src/lib-auth/auth-server-connection.c
+++ b/src/lib-auth/auth-server-connection.c
@@ -453,9 +453,10 @@ int auth_server_connection_connect(struct auth_server_connection *conn)
                                     AUTH_CLIENT_PROTOCOL_MINOR_VERSION,
 				    conn->client->client_pid);
 	if (o_stream_send_str(conn->output, handshake) < 0) {
-		i_warning("Error sending handshake to auth server: %m");
+		i_warning("Error sending handshake to auth server: %s",
+			  o_stream_get_error(conn->output));
 		auth_server_connection_disconnect(conn,
-			strerror(conn->output->last_failed_errno));
+			o_stream_get_error(conn->output));
 		return -1;
 	}
 
diff --git a/src/lib-http/http-server-response.c b/src/lib-http/http-server-response.c
index 42f328cd836..aa976640a6d 100644
--- a/src/lib-http/http-server-response.c
+++ b/src/lib-http/http-server-response.c
@@ -142,8 +142,8 @@ void http_server_response_set_payload(struct http_server_response *resp,
 	resp->payload_input = input;
 	if ((ret = i_stream_get_size(input, TRUE, &resp->payload_size)) <= 0) {
 		if (ret < 0) {
-			i_error("i_stream_get_size(%s) failed: %m",
-				i_stream_get_name(input));
+			i_error("i_stream_get_size(%s) failed: %s",
+				i_stream_get_name(input), i_stream_get_error(input));
 		}
 		resp->payload_size = 0;
 		resp->payload_chunked = TRUE;
@@ -500,16 +500,16 @@ int http_server_response_send_more(struct http_server_response *resp,
 	if (resp->payload_input->stream_errno != 0) {
 		/* we're in the middle of sending a response, so the connection
 		   will also have to be aborted */
-		errno = resp->payload_input->stream_errno;
-		*error_r = t_strdup_printf("read(%s) failed: %m",
-					   i_stream_get_name(resp->payload_input));
+		*error_r = t_strdup_printf("read(%s) failed: %s",
+			i_stream_get_name(resp->payload_input),
+			i_stream_get_error(resp->payload_input));
 		ret = -1;
 	} else if (output->stream_errno != 0) {
 		/* failed to send response */
-		errno = output->stream_errno;
-		if (errno != EPIPE && errno != ECONNRESET) {
-			*error_r = t_strdup_printf("write(%s) failed: %m",
-					   o_stream_get_name(output));
+		if (output->stream_errno != EPIPE &&
+		    output->stream_errno != ECONNRESET) {
+			*error_r = t_strdup_printf("write(%s) failed: %s",
+				o_stream_get_name(output), o_stream_get_error(output));
 		}
 		ret = -1;
 	} else {
@@ -646,9 +646,10 @@ static int http_server_response_send_real(struct http_server_response *resp,
 	o_stream_ref(output);
 	o_stream_cork(output);
 	if (o_stream_sendv(output, iov, N_ELEMENTS(iov)) < 0) {
-		if (errno != EPIPE && errno != ECONNRESET) {
-			*error_r = t_strdup_printf("write(%s) failed: %m",
-					   o_stream_get_name(output));
+		if (output->stream_errno != EPIPE &&
+		    output->stream_errno != ECONNRESET) {
+			*error_r = t_strdup_printf("write(%s) failed: %s",
+				o_stream_get_name(output), o_stream_get_error(output));
 		}
 		ret = -1;
 	}
diff --git a/src/lib-index/mail-index-strmap.c b/src/lib-index/mail-index-strmap.c
index e55cc2b32f8..7360e259245 100644
--- a/src/lib-index/mail-index-strmap.c
+++ b/src/lib-index/mail-index-strmap.c
@@ -1018,8 +1018,8 @@ static int mail_index_strmap_recreate(struct mail_index_strmap_view *view)
 	o_stream_cork(output);
 	mail_index_strmap_recreate_write(view, output);
 	if (o_stream_nfinish(output) < 0) {
-		mail_index_set_error(strmap->index,
-				     "write(%s) failed: %m", temp_path);
+		mail_index_set_error(strmap->index, "write(%s) failed: %s",
+				     temp_path, o_stream_get_error(output));
 		ret = -1;
 	}
 	o_stream_destroy(&output);
diff --git a/src/lib-lda/duplicate.c b/src/lib-lda/duplicate.c
index b63dcbdca53..65040c408e4 100644
--- a/src/lib-lda/duplicate.c
+++ b/src/lib-lda/duplicate.c
@@ -318,7 +318,8 @@ void duplicate_flush(struct duplicate_context *ctx)
 	hash_table_iterate_deinit(&iter);
 
 	if (o_stream_nfinish(output) < 0) {
-		i_error("write(%s) failed: %m", file->path);
+		i_error("write(%s) failed: %s", file->path,
+			o_stream_get_error(output));
 		o_stream_unref(&output);
 		duplicate_file_free(&ctx->file);
 		return;
diff --git a/src/lib-lda/lmtp-client.c b/src/lib-lda/lmtp-client.c
index e47e0e8b6e8..239c6a6c46d 100644
--- a/src/lib-lda/lmtp-client.c
+++ b/src/lib-lda/lmtp-client.c
@@ -604,8 +604,8 @@ static void lmtp_client_input(struct lmtp_client *client)
 		lmtp_client_fail(client,
 				 "501 5.5.4 Command reply line too long");
 	} else if (client->input->stream_errno != 0) {
-		errno = client->input->stream_errno;
-		i_error("lmtp client: read() failed: %m");
+		i_error("lmtp client: read() failed: %s",
+			i_stream_get_error(client->input));
 		lmtp_client_fail(client, ERRSTR_TEMP_REMOTE_FAILURE
 				 " (read failure)");
 	} else if (client->input->eof) {
diff --git a/src/lib-lda/smtp-client.c b/src/lib-lda/smtp-client.c
index 28bb1eb3b22..131766ef038 100644
--- a/src/lib-lda/smtp-client.c
+++ b/src/lib-lda/smtp-client.c
@@ -265,14 +265,14 @@ smtp_client_send_flush(struct smtp_client *smtp_client,
 	}
 
 	if (o_stream_nfinish(smtp_client->output) < 0) {
-		*error_r = t_strdup_printf("write(%s) failed: %m",
-					   smtp_client->temp_path);
+		*error_r = t_strdup_printf("write(%s) failed: %s",
+			smtp_client->temp_path, o_stream_get_error(smtp_client->output));
 		return -1;
 	}
 
 	if (o_stream_seek(smtp_client->output, 0) < 0) {
-		*error_r = t_strdup_printf("lseek(%s) failed: %m",
-					   smtp_client->temp_path);
+		*error_r = t_strdup_printf("lseek(%s) failed: %s",
+			smtp_client->temp_path, o_stream_get_error(smtp_client->output));
 		return -1;
 	}
 
diff --git a/src/lib-master/anvil-client.c b/src/lib-master/anvil-client.c
index 553b21b041e..eff41ff5780 100644
--- a/src/lib-master/anvil-client.c
+++ b/src/lib-master/anvil-client.c
@@ -113,7 +113,8 @@ static void anvil_input(struct anvil_client *client)
 		aqueue_delete_tail(client->queries);
 	}
 	if (client->input->stream_errno != 0) {
-		i_error("read(%s) failed: %m", client->path);
+		i_error("read(%s) failed: %s", client->path,
+			i_stream_get_error(client->input));
 		anvil_reconnect(client);
 	} else if (client->input->eof) {
 		i_error("read(%s) failed: EOF", client->path);
diff --git a/src/lib-master/master-instance.c b/src/lib-master/master-instance.c
index b05ecb29008..271adfe2782 100644
--- a/src/lib-master/master-instance.c
+++ b/src/lib-master/master-instance.c
@@ -116,7 +116,7 @@ static int master_instance_list_refresh(struct master_instance_list *list)
 			i_error("Invalid line in %s: %s", list->path, line);
 	} T_END;
 	if (input->stream_errno != 0) {
-		i_error("read(%s) failed: %m", line);
+		i_error("read(%s) failed: %s", line, i_stream_get_error(input));
 		ret = -1;
 	}
 	i_stream_destroy(&input);
@@ -147,7 +147,7 @@ master_instance_list_write(struct master_instance_list *list,
 		o_stream_nsend(output, str_data(str), str_len(str));
 	}
 	if (o_stream_nfinish(output) < 0) {
-		i_error("write(%s) failed: %m", path);
+		i_error("write(%s) failed: %s", path, o_stream_get_error(output));
 		ret = -1;
 	}
 	o_stream_destroy(&output);
diff --git a/src/lib-master/master-service-settings.c b/src/lib-master/master-service-settings.c
index 5d02c2b80b4..4a25f6592cc 100644
--- a/src/lib-master/master-service-settings.c
+++ b/src/lib-master/master-service-settings.c
@@ -327,7 +327,8 @@ config_read_reply_header(struct istream *istream, const char *path, pool_t pool,
 		if (ret == 0)
 			return 1;
 		*error_r = istream->stream_errno != 0 ?
-			t_strdup_printf("read(%s) failed: %m", path) :
+			t_strdup_printf("read(%s) failed: %s", path,
+					i_stream_get_error(istream)) :
 			t_strdup_printf("read(%s) failed: EOF", path);
 		return -1;
 	}
diff --git a/src/lib-storage/index/cydir/cydir-save.c b/src/lib-storage/index/cydir/cydir-save.c
index cc964cd9b03..1d8b696c65d 100644
--- a/src/lib-storage/index/cydir/cydir-save.c
+++ b/src/lib-storage/index/cydir/cydir-save.c
@@ -162,7 +162,8 @@ static int cydir_save_flush(struct cydir_save_context *ctx, const char *path)
 	int ret = 0;
 
 	if (o_stream_nfinish(ctx->ctx.data.output) < 0) {
-		mail_storage_set_critical(storage, "write(%s) failed: %m", path);
+		mail_storage_set_critical(storage, "write(%s) failed: %s", path,
+			o_stream_get_error(ctx->ctx.data.output));
 		ret = -1;
 	}
 
diff --git a/src/lib-storage/index/dbox-multi/mdbox-purge.c b/src/lib-storage/index/dbox-multi/mdbox-purge.c
index 8b749f1b073..8921a0b28a1 100644
--- a/src/lib-storage/index/dbox-multi/mdbox-purge.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-purge.c
@@ -78,7 +78,8 @@ mdbox_file_read_metadata_hdr(struct dbox_file *file,
 			return 0;
 		}
 		mail_storage_set_critical(&file->storage->storage,
-			"read(%s) failed: %m", file->cur_path);
+			"read(%s) failed: %s", file->cur_path,
+			i_stream_get_error(file->input));
 		return -1;
 	}
 
@@ -192,7 +193,6 @@ mdbox_purge_save_msg(struct mdbox_purge_context *ctx, struct dbox_file *file,
 	enum mdbox_map_append_flags append_flags;
 	uoff_t msg_size;
 	off_t ret;
-	int read_errno;
 
 	if (ctx->append_ctx == NULL)
 		ctx->append_ctx = mdbox_map_append_begin(ctx->atomic);
@@ -208,19 +208,19 @@ mdbox_purge_save_msg(struct mdbox_purge_context *ctx, struct dbox_file *file,
 
 	input = i_stream_create_limit(file->input, msg_size);
 	ret = o_stream_send_istream(output, input);
-	read_errno = input->stream_errno;
-	i_stream_unref(&input);
-
-	if (read_errno != 0) {
-		errno = read_errno;
+	if (input->stream_errno != 0) {
 		mail_storage_set_critical(&file->storage->storage,
-			"read(%s) failed: %m", file->cur_path);
+			"read(%s) failed: %s", file->cur_path,
+			i_stream_get_error(input));
+		i_stream_unref(&input);
 		return -1;
 	}
+	i_stream_unref(&input);
 	if (o_stream_nfinish(output) < 0) {
 		mail_storage_set_critical(&file->storage->storage,
-					  "write(%s) failed: %m",
-					  out_file_append->file->cur_path);
+					  "write(%s) failed: %s",
+					  out_file_append->file->cur_path,
+					  o_stream_get_error(output));
 		return -1;
 	}
 	if (ret != (off_t)msg_size) {
diff --git a/src/lib-storage/index/dbox-single/sdbox-file.c b/src/lib-storage/index/dbox-single/sdbox-file.c
index ceb60738a58..8c7d7a5632c 100644
--- a/src/lib-storage/index/dbox-single/sdbox-file.c
+++ b/src/lib-storage/index/dbox-single/sdbox-file.c
@@ -328,20 +328,15 @@ int sdbox_file_move(struct dbox_file *file, bool alt_path)
 
 	output = o_stream_create_fd_file(out_fd, 0, FALSE);
 	i_stream_seek(file->input, 0);
-	while ((ret = o_stream_send_istream(output, file->input)) > 0) ;
+	ret = o_stream_send_istream(output, file->input) > 0 ? 0 : -1;
 	if (o_stream_nfinish(output) < 0) {
-		mail_storage_set_critical(storage, "write(%s) failed: %m",
-					  temp_path);
+		mail_storage_set_critical(storage, "write(%s) failed: %s",
+			temp_path, o_stream_get_error(output));
 		ret = -1;
 	} else if (file->input->stream_errno != 0) {
-		errno = file->input->stream_errno;
-		dbox_file_set_syscall_error(file, "ftruncate()");
+		mail_storage_set_critical(storage, "read(%s) failed: %s",
+			temp_path, i_stream_get_error(file->input));
 		ret = -1;
-	} else if (ret < 0) {
-		mail_storage_set_critical(storage,
-			"o_stream_send_istream(%s, %s) "
-			"failed with unknown error",
-			temp_path, file->cur_path);
 	}
 	o_stream_unref(&output);
 
diff --git a/src/lib-storage/index/imapc/imapc-save.c b/src/lib-storage/index/imapc/imapc-save.c
index e6012ff8d8e..89f394c0457 100644
--- a/src/lib-storage/index/imapc/imapc-save.c
+++ b/src/lib-storage/index/imapc/imapc-save.c
@@ -280,7 +280,8 @@ int imapc_save_finish(struct mail_save_context *_ctx)
 		if (o_stream_nfinish(_ctx->data.output) < 0) {
 			if (!mail_storage_set_error_from_errno(storage)) {
 				mail_storage_set_critical(storage,
-					"write(%s) failed: %m", ctx->temp_path);
+					"write(%s) failed: %s", ctx->temp_path,
+					o_stream_get_error(_ctx->data.output));
 			}
 			ctx->failed = TRUE;
 		}
diff --git a/src/lib-storage/index/index-attachment.c b/src/lib-storage/index/index-attachment.c
index 5ec3efd042d..4b7abd9129d 100644
--- a/src/lib-storage/index/index-attachment.c
+++ b/src/lib-storage/index/index-attachment.c
@@ -196,10 +196,9 @@ static int save_check_write_error(struct mail_storage *storage,
 	if (output->last_failed_errno == 0)
 		return 0;
 
-	errno = output->last_failed_errno;
 	if (!mail_storage_set_error_from_errno(storage)) {
-		mail_storage_set_critical(storage, "write(%s) failed: %m",
-					  o_stream_get_name(output));
+		mail_storage_set_critical(storage, "write(%s) failed: %s",
+			o_stream_get_name(output), o_stream_get_error(output));
 	}
 	return -1;
 }
diff --git a/src/lib-storage/index/maildir/maildir-save.c b/src/lib-storage/index/maildir/maildir-save.c
index 228c2721425..2ee201df9ec 100644
--- a/src/lib-storage/index/maildir/maildir-save.c
+++ b/src/lib-storage/index/maildir/maildir-save.c
@@ -536,7 +536,7 @@ static int maildir_save_finish_real(struct mail_save_context *_ctx)
 {
 	struct maildir_save_context *ctx = (struct maildir_save_context *)_ctx;
 	struct mail_storage *storage = &ctx->mbox->storage->storage;
-	const char *path;
+	const char *path, *output_errstr;
 	off_t real_size;
 	uoff_t size;
 	int output_errno;
@@ -551,7 +551,8 @@ static int maildir_save_finish_real(struct mail_save_context *_ctx)
 	if (!ctx->failed && o_stream_nfinish(_ctx->data.output) < 0) {
 		if (!mail_storage_set_error_from_errno(storage)) {
 			mail_storage_set_critical(storage,
-				"write(%s) failed: %m", path);
+				"write(%s) failed: %s", path,
+				o_stream_get_error(_ctx->data.output));
 		}
 		ctx->failed = TRUE;
 	}
@@ -582,6 +583,7 @@ static int maildir_save_finish_real(struct mail_save_context *_ctx)
 		ctx->file_last->vsize = (uoff_t)-1;
 
 	output_errno = _ctx->data.output->last_failed_errno;
+	output_errstr = t_strdup(o_stream_get_error(_ctx->data.output));
 	o_stream_destroy(&_ctx->data.output);
 
 	if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER &&
@@ -625,13 +627,12 @@ static int maildir_save_finish_real(struct mail_save_context *_ctx)
 		/* delete the tmp file */
 		i_unlink_if_exists(path);
 
-		errno = output_errno;
-		if (ENOQUOTA(errno)) {
+		if (ENOQUOTA(output_errno)) {
 			mail_storage_set_error(storage,
 				MAIL_ERROR_NOQUOTA, MAIL_ERRSTR_NO_QUOTA);
-		} else if (errno != 0) {
+		} else if (output_errno != 0) {
 			mail_storage_set_critical(storage,
-				"write(%s) failed: %m", path);
+				"write(%s) failed: %s", path, output_errstr);
 		}
 
 		maildir_save_remove_last_filename(ctx);
diff --git a/src/lib-storage/index/maildir/maildir-uidlist.c b/src/lib-storage/index/maildir/maildir-uidlist.c
index eb89aa558d4..b0208f2672e 100644
--- a/src/lib-storage/index/maildir/maildir-uidlist.c
+++ b/src/lib-storage/index/maildir/maildir-uidlist.c
@@ -832,9 +832,9 @@ maildir_uidlist_update_read(struct maildir_uidlist *uidlist,
                 if (input->stream_errno == ESTALE && try_retry)
 			*retry_r = TRUE;
 		else {
-			errno = input->stream_errno;
 			mail_storage_set_critical(storage,
-				"read(%s) failed: %m", uidlist->path);
+				"read(%s) failed: %s", uidlist->path,
+				i_stream_get_error(input));
 		}
 		uidlist->last_read_offset = 0;
 	}
@@ -1312,7 +1312,8 @@ static int maildir_uidlist_write_fd(struct maildir_uidlist *uidlist, int fd,
 	maildir_uidlist_iter_deinit(&iter);
 
 	if (o_stream_nfinish(output) < 0) {
-		mail_storage_set_critical(storage, "write(%s) failed: %m", path);
+		mail_storage_set_critical(storage, "write(%s) failed: %s", path,
+					  o_stream_get_error(output));
 		o_stream_unref(&output);
 		return -1;
 	}
diff --git a/src/lib/json-parser.c b/src/lib/json-parser.c
index fe54a19cdf1..4e6afbf85c1 100644
--- a/src/lib/json-parser.c
+++ b/src/lib/json-parser.c
@@ -133,8 +133,9 @@ int json_parser_deinit(struct json_parser **_parser, const char **error_r)
 		/* actual parser error */
 		*error_r = parser->error;
 	} else if (parser->input->stream_errno != 0) {
-		*error_r = t_strdup_printf("read(%s) failed: %m",
-					   i_stream_get_name(parser->input));
+		*error_r = t_strdup_printf("read(%s) failed: %s",
+					   i_stream_get_name(parser->input),
+					   i_stream_get_error(parser->input));
 	} else if (parser->data == parser->end &&
 		   !i_stream_have_bytes_left(parser->input) &&
 		   parser->state != JSON_STATE_DONE) {
diff --git a/src/plugins/acl/acl-backend-vfile-acllist.c b/src/plugins/acl/acl-backend-vfile-acllist.c
index 28764171b07..6e3fd226d98 100644
--- a/src/plugins/acl/acl-backend-vfile-acllist.c
+++ b/src/plugins/acl/acl-backend-vfile-acllist.c
@@ -279,7 +279,8 @@ acl_backend_vfile_acllist_try_rebuild(struct acl_backend_vfile *backend)
 	}
 
 	if (o_stream_nfinish(output) < 0) {
-		i_error("write(%s) failed: %m", str_c(path));
+		i_error("write(%s) failed: %s", str_c(path),
+			o_stream_get_error(output));
 		ret = -1;
 	}
 	if (mailbox_list_iter_deinit(&iter) < 0)
diff --git a/src/plugins/acl/acl-backend-vfile-update.c b/src/plugins/acl/acl-backend-vfile-update.c
index ecd2dea99db..0c2ebadee07 100644
--- a/src/plugins/acl/acl-backend-vfile-update.c
+++ b/src/plugins/acl/acl-backend-vfile-update.c
@@ -159,7 +159,8 @@ acl_backend_vfile_update_write(struct acl_object *aclobj,
 	}
 	str_free(&str);
 	if (o_stream_nfinish(output) < 0) {
-		i_error("write(%s) failed: %m", path);
+		i_error("write(%s) failed: %s", path,
+			o_stream_get_error(output));
 		ret = -1;
 	}
 	o_stream_destroy(&output);
diff --git a/src/plugins/acl/acl-backend-vfile.c b/src/plugins/acl/acl-backend-vfile.c
index 188a41cbaaa..db8048a5df7 100644
--- a/src/plugins/acl/acl-backend-vfile.c
+++ b/src/plugins/acl/acl-backend-vfile.c
@@ -393,7 +393,8 @@ acl_backend_vfile_read(struct acl_object *aclobj, bool global, const char *path,
 			ret = 0;
 		else {
 			ret = -1;
-			i_error("read(%s) failed: %m", path);
+			i_error("read(%s) failed: %s", path,
+				i_stream_get_error(input));
 		}
 	} else {
 		if (fstat(fd, &st) < 0) {
diff --git a/src/plugins/fts-squat/squat-trie.c b/src/plugins/fts-squat/squat-trie.c
index 0861f21b0c8..b3d8fa301b8 100644
--- a/src/plugins/fts-squat/squat-trie.c
+++ b/src/plugins/fts-squat/squat-trie.c
@@ -1680,7 +1680,8 @@ static int squat_trie_write(struct squat_trie_build_context *ctx)
 		o_stream_nsend(output, &trie->hdr, sizeof(trie->hdr));
 	}
 	if (o_stream_nfinish(output) < 0) {
-		i_error("write() to %s failed: %m", path);
+		i_error("write(%s) failed: %s", path,
+			o_stream_get_error(output));
 		ret = -1;
 	}
 	o_stream_destroy(&output);
diff --git a/src/plugins/fts-squat/squat-uidlist.c b/src/plugins/fts-squat/squat-uidlist.c
index 92bc86fa469..1b7fb1cad19 100644
--- a/src/plugins/fts-squat/squat-uidlist.c
+++ b/src/plugins/fts-squat/squat-uidlist.c
@@ -861,7 +861,8 @@ int squat_uidlist_build_finish(struct squat_uidlist_build_context *ctx)
 	}
 
 	if (o_stream_nfinish(ctx->output) < 0) {
-		i_error("write() to %s failed: %m", ctx->uidlist->path);
+		i_error("write() to %s failed: %s", ctx->uidlist->path,
+			o_stream_get_error(ctx->output));
 		return -1;
 	}
 	return 0;
@@ -1064,7 +1065,8 @@ int squat_uidlist_rebuild_finish(struct squat_uidlist_rebuild_context *ctx,
 		if (ctx->uidlist->corrupted)
 			ret = -1;
 		else if (o_stream_nfinish(ctx->output) < 0) {
-			i_error("write() to %s failed: %m", temp_path);
+			i_error("write(%s) failed: %s", temp_path,
+				o_stream_get_error(ctx->output));
 			ret = -1;
 		} else if (rename(temp_path, ctx->uidlist->path) < 0) {
 			i_error("rename(%s, %s) failed: %m",
diff --git a/src/plugins/fts/fts-expunge-log.c b/src/plugins/fts/fts-expunge-log.c
index 27ce54ee88e..0deb7bed270 100644
--- a/src/plugins/fts/fts-expunge-log.c
+++ b/src/plugins/fts/fts-expunge-log.c
@@ -424,7 +424,8 @@ fts_expunge_log_read_failure(struct fts_expunge_log_read_ctx *ctx,
 
 	if (ctx->input->stream_errno != 0) {
 		ctx->failed = TRUE;
-		i_error("read(%s) failed: %m", ctx->log->path);
+		i_error("read(%s) failed: %s", ctx->log->path,
+			i_stream_get_error(ctx->input));
 	} else {
 		size = i_stream_get_data_size(ctx->input);
 		ctx->corrupted = TRUE;

From 9aaf79f7c7b38b1104e5e04f5f9c58194c3ff477 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 16 May 2016 19:16:55 +0300
Subject: [PATCH 237/450] lmtp: Fixed error logging when writing to temp file

---
 src/lmtp/commands.c | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/lmtp/commands.c b/src/lmtp/commands.c
index 5d3f9c3e5fb..5528de970a5 100644
--- a/src/lmtp/commands.c
+++ b/src/lmtp/commands.c
@@ -1171,7 +1171,6 @@ static int client_input_add_file(struct client *client,
 {
 	struct client_state *state = &client->state;
 	string_t *path;
-	ssize_t ret;
 	int fd;
 
 	if (state->mail_data_output != NULL) {
@@ -1200,15 +1199,17 @@ static int client_input_add_file(struct client *client,
 
 	state->mail_data_fd = fd;
 	state->mail_data_output = o_stream_create_fd_file(fd, 0, FALSE);
+	o_stream_set_name(state->mail_data_output, str_c(path));
 	o_stream_cork(state->mail_data_output);
 
-	ret = o_stream_send(state->mail_data_output,
-			    state->mail_data->data, state->mail_data->used);
-	if (ret != (ssize_t)state->mail_data->used)
-		return -1;
-	if (o_stream_send(client->state.mail_data_output,
-			  data, size) != (ssize_t)size)
+	o_stream_nsend(state->mail_data_output,
+		       state->mail_data->data, state->mail_data->used);
+	o_stream_nsend(client->state.mail_data_output, data, size);
+	if (o_stream_nfinish(client->state.mail_data_output) < 0) {
+		i_error("write(%s) failed: %s", str_c(path),
+			o_stream_get_error(client->state.mail_data_output));
 		return -1;
+	}
 	return 0;
 }
 

From 3a349ae342efc02c454d2c9a8c508a4b3d253094 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 18 May 2016 18:41:33 +0300
Subject: [PATCH 238/450] lib: ostream-escaped: Fixed setting offset

---
 src/lib/ostream-escaped.c      | 14 +++++---------
 src/lib/test-ostream-escaped.c |  7 +++++--
 2 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/src/lib/ostream-escaped.c b/src/lib/ostream-escaped.c
index c875d70c7ee..b3613ca32b4 100644
--- a/src/lib/ostream-escaped.c
+++ b/src/lib/ostream-escaped.c
@@ -40,7 +40,7 @@ static ssize_t
 o_stream_escaped_send_chunk(struct escaped_ostream *estream,
 			    const unsigned char *data, size_t len)
 {
-	size_t i, max_buffer_size, flush_pos;
+	size_t i, max_buffer_size;
 	ssize_t ret;
 
 	max_buffer_size = I_MIN(o_stream_get_max_buffer_size(estream->ostream.parent),
@@ -50,24 +50,20 @@ o_stream_escaped_send_chunk(struct escaped_ostream *estream,
 		max_buffer_size = IO_BLOCK_SIZE;
 	}
 
-	flush_pos = str_len(estream->buf);
 	for (i = 0; i < len; i++) {
 		if (str_len(estream->buf) + 2 > max_buffer_size) { /* escaping takes at least two bytes */
-			estream->ostream.ostream.offset +=
-				str_len(estream->buf) - flush_pos;
 			ret = o_stream_escaped_send_outbuf(estream);
-			if (ret < 0)
+			if (ret < 0) {
+				estream->ostream.ostream.offset += i;
 				return ret;
-			flush_pos = str_len(estream->buf);
+			}
 			if (ret == 0)
 				break;
 		}
 		estream->format(estream->buf, data[i]);
 		estream->flushed = FALSE;
 	}
-	/* we'll return how many bytes of input we consumed, but ostream offset
-	   contains how many bytes we actually wrote */
-	estream->ostream.ostream.offset += str_len(estream->buf) - flush_pos;
+	estream->ostream.ostream.offset += i;
 	return i;
 }
 
diff --git a/src/lib/test-ostream-escaped.c b/src/lib/test-ostream-escaped.c
index f64813f4a0c..29d0235d9a8 100644
--- a/src/lib/test-ostream-escaped.c
+++ b/src/lib/test-ostream-escaped.c
@@ -24,6 +24,7 @@ static void test_ostream_escaped_json(void)
 	iov[1].iov_len = 7;
 	test_assert(o_stream_sendv(os_encode, iov, 2) == 12);
 	test_assert(os_encode->offset == 12);
+	test_assert(os_sink->offset == 12);
 	test_assert(strcmp(str_c(str), "hello, world") == 0);
 
 	/* reset buffer */
@@ -33,7 +34,8 @@ static void test_ostream_escaped_json(void)
 	o_stream_set_max_buffer_size(os_encode, 10);
 	o_stream_set_max_buffer_size(os_sink, 100);
 	test_assert(o_stream_send(os_encode, "\x15\x00!\x00\x15\x11" "123456", 12) == 12);
-	test_assert(os_encode->offset == 2*6 + 1 + 3*6 + 6);
+	test_assert(os_encode->offset == 12);
+	test_assert(os_sink->offset == 2*6 + 1 + 3*6 + 6);
 	test_assert(strcmp(str_c(str), "\\u0015\\u0000!\\u0000\\u0015\\u0011123456") == 0);
 
 	/* reset buffer */
@@ -49,7 +51,8 @@ static void test_ostream_escaped_json(void)
 	o_stream_set_max_buffer_size(os_sink, 100);
 	ret += o_stream_send_str(os_encode, partial_input + ret);
 	test_assert(ret == (ssize_t)strlen(partial_input));
-	test_assert(os_encode->offset == str_len(str));
+	test_assert((ssize_t)os_encode->offset == ret);
+	test_assert(os_sink->offset == str_len(str));
 	test_assert(strcmp(str_c(str), "\\u0015!\\u0001?#&") == 0);
 
 	o_stream_unref(&os_encode);

From 7283d40b1d26c5a951a61a4cb2f14159b081d4eb Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 18 May 2016 14:24:55 +0300
Subject: [PATCH 239/450] lib-imap-storage: Minor error logging cleanup/fix.

We read from input, not from mail_input. In theory it could have the actual
error instead of mail_input (although right now it couldn't).
---
 src/lib-imap-storage/imap-msgpart.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib-imap-storage/imap-msgpart.c b/src/lib-imap-storage/imap-msgpart.c
index 37e85b9ee9a..3c719eeaba5 100644
--- a/src/lib-imap-storage/imap-msgpart.c
+++ b/src/lib-imap-storage/imap-msgpart.c
@@ -396,8 +396,8 @@ imap_msgpart_get_partial_header(struct mail *mail, struct istream *mail_input,
 	if (message_get_header_size(input, &hdr_size, &has_nuls) < 0) {
 		errno = input->stream_errno;
 		mail_storage_set_critical(mail->box->storage,
-			"read(%s) failed: %s", i_stream_get_name(mail_input),
-			i_stream_get_error(mail_input));
+			"read(%s) failed: %s", i_stream_get_name(input),
+			i_stream_get_error(input));
 		i_stream_unref(&input);
 		return -1;
 	}

From 169c8862c63aaffad634961241ea3d0df7825f39 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 18 May 2016 21:41:49 +0300
Subject: [PATCH 240/450] lib-imap-storage: Don't sort uninitialized array on
 invalid header list.

---
 src/lib-imap-storage/imap-msgpart.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/lib-imap-storage/imap-msgpart.c b/src/lib-imap-storage/imap-msgpart.c
index 3c719eeaba5..7eeabb0fafe 100644
--- a/src/lib-imap-storage/imap-msgpart.c
+++ b/src/lib-imap-storage/imap-msgpart.c
@@ -166,13 +166,12 @@ imap_msgpart_get_header_fields(pool_t pool, const char *header_list,
 			value = p_strdup(pool, t_str_ucase(value));
 			array_append(fields, &value, 1);
 		}
+		/* istream-header-filter requires headers to be sorted */
+		array_sort(fields, i_strcasecmp_p);
 	} else {
 		result = -1;
 	}
 
-	/* istream-header-filter requires headers to be sorted */
-	array_sort(fields, i_strcasecmp_p);
-
 	imap_parser_unref(&parser);
 	i_stream_unref(&input);
 	return result;

From 789f321c28d9113b3b8e9766792b397a99d59fb5 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 23 May 2016 17:47:50 +0300
Subject: [PATCH 241/450] auth: Support %variable expansion for LDAP field
 names.

For example this is now allowed:

user_attrs = \
  =namespace/%{ldap:enabledNamespace}/enabled=yes \
  ...
---
 src/auth/db-ldap.c | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/src/auth/db-ldap.c b/src/auth/db-ldap.c
index 3a8c3f51092..f5f974d9c94 100644
--- a/src/auth/db-ldap.c
+++ b/src/auth/db-ldap.c
@@ -1552,6 +1552,7 @@ db_ldap_result_iterate_init_full(struct ldap_connection *conn,
 	ctx->skip_null_values = skip_null_values;
 	ctx->iter_dn_values = iter_dn_values;
 	hash_table_create(&ctx->ldap_attrs, pool, 0, strcase_hash, strcasecmp);
+	ctx->var = str_new(ctx->pool, 256);
 	if (ctx->auth_request->debug)
 		ctx->debug = t_str_new(256);
 
@@ -1630,16 +1631,17 @@ static const char *db_ldap_field_ptr_expand(const char *data, void *context)
 	return db_ldap_field_expand(field_name, ctx);
 }
 
+static struct var_expand_func_table ldap_var_funcs_table[] = {
+	{ "ldap", db_ldap_field_expand },
+	{ "ldap_ptr", db_ldap_field_ptr_expand },
+	{ NULL, NULL }
+};
+
 static const char *const *
 db_ldap_result_return_value(struct db_ldap_result_iterate_context *ctx,
 			    const struct ldap_field *field,
 			    struct db_ldap_value *ldap_value)
 {
-	static struct var_expand_func_table var_funcs_table[] = {
-		{ "ldap", db_ldap_field_expand },
-		{ "ldap_ptr", db_ldap_field_ptr_expand },
-		{ NULL, NULL }
-	};
 	const struct var_expand_table *var_table;
 	const char *const *values;
 
@@ -1673,12 +1675,8 @@ db_ldap_result_return_value(struct db_ldap_result_iterate_context *ctx,
 		      (and less importantly the same for other variables) */
 		var_table = db_ldap_value_get_var_expand_table(ctx->auth_request,
 							       values[0]);
-		if (ctx->var == NULL)
-			ctx->var = str_new(ctx->pool, 256);
-		else
-			str_truncate(ctx->var, 0);
 		var_expand_with_funcs(ctx->var, field->value, var_table,
-				      var_funcs_table, ctx);
+				      ldap_var_funcs_table, ctx);
 		ctx->val_1_arr[0] = str_c(ctx->var);
 		values = ctx->val_1_arr;
 	}
@@ -1691,6 +1689,7 @@ bool db_ldap_result_iterate_next(struct db_ldap_result_iterate_context *ctx,
 {
 	const struct ldap_field *field;
 	struct db_ldap_value *ldap_value;
+	unsigned int pos;
 
 	do {
 		if (ctx->attr_idx == array_count(ctx->attr_map))
@@ -1706,9 +1705,23 @@ bool db_ldap_result_iterate_next(struct db_ldap_result_iterate_context *ctx,
 	else if (ctx->debug && *field->ldap_attr_name != '\0')
 		str_printfa(ctx->debug, "; %s missing", field->ldap_attr_name);
 
-	*name_r = field->name;
+	str_truncate(ctx->var, 0);
 	*values_r = db_ldap_result_return_value(ctx, field, ldap_value);
 
+	if (strchr(field->name, '%') == NULL)
+		*name_r = field->name;
+	else {
+		/* expand %variables also for LDAP name fields. we'll use the
+		   same ctx->var, which may already contain the value. */
+		str_append_c(ctx->var, '\0');
+		pos = str_len(ctx->var);
+
+		var_expand_with_funcs(ctx->var, field->name,
+			auth_request_get_var_expand_table(ctx->auth_request, NULL),
+			ldap_var_funcs_table, ctx);
+		*name_r = str_c(ctx->var) + pos;
+	}
+
 	if (ctx->skip_null_values && (*values_r)[0] == NULL) {
 		/* no values. don't confuse the caller with this reply. */
 		return db_ldap_result_iterate_next(ctx, name_r, values_r);

From 406b8be33b563f081ffc229534d7bf49e0fc5174 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 2 Jun 2016 16:06:08 +0300
Subject: [PATCH 242/450] dict-sql: Improve error message for invalid value
 fields.

It'll now show which map's pattern matched, making it easier to find
from the config file.
---
 src/lib-dict/dict-sql.c | 32 ++++++++++++++++++--------------
 1 file changed, 18 insertions(+), 14 deletions(-)

diff --git a/src/lib-dict/dict-sql.c b/src/lib-dict/dict-sql.c
index 4a0a140ae66..f66d0a6047b 100644
--- a/src/lib-dict/dict-sql.c
+++ b/src/lib-dict/dict-sql.c
@@ -214,6 +214,7 @@ sql_dict_find_map(struct sql_dict *dict, const char *path,
 
 static int
 sql_dict_value_escape(string_t *str, struct sql_dict *dict,
+		      const struct dict_sql_map *map,
 		      enum dict_sql_type value_type, const char *field_name,
 		      const char *value, const char *value_suffix,
 		      const char **error_r)
@@ -229,8 +230,8 @@ sql_dict_value_escape(string_t *str, struct sql_dict *dict,
 	case DICT_SQL_TYPE_UINT:
 		if (value_suffix[0] != '\0' || str_to_uint(value, &num) < 0) {
 			*error_r = t_strdup_printf(
-				"field %s value isn't unsigned integer: %s%s",
-				field_name, value, value_suffix);
+				"%s field's value isn't unsigned integer: %s%s (in pattern: %s)",
+				field_name, value, value_suffix, map->pattern);
 			return -1;
 		}
 		str_printfa(str, "%u", num);
@@ -243,8 +244,8 @@ sql_dict_value_escape(string_t *str, struct sql_dict *dict,
 	if (hex_to_binary(value, buf) < 0) {
 		/* we shouldn't get untrusted input here. it's also a bit
 		   annoying to handle this error. */
-		*error_r = t_strdup_printf("field %s value isn't hexblob: %s",
-					   field_name, value);
+		*error_r = t_strdup_printf("%s field's value isn't hexblob: %s (in pattern: %s)",
+					   field_name, value, map->pattern);
 		return -1;
 	}
 	str_append(buf, value_suffix);
@@ -254,11 +255,12 @@ sql_dict_value_escape(string_t *str, struct sql_dict *dict,
 
 static int
 sql_dict_field_escape_value(string_t *str, struct sql_dict *dict,
+			    const struct dict_sql_map *map,
 			    const struct dict_sql_field *field,
 			    const char *value, const char *value_suffix,
 			    const char **error_r)
 {
-	return sql_dict_value_escape(str, dict, field->value_type,
+	return sql_dict_value_escape(str, dict, map, field->value_type,
 				     field->name, value, value_suffix, error_r);
 }
 
@@ -290,7 +292,7 @@ sql_dict_where_build(struct sql_dict *dict, const struct dict_sql_map *map,
 		if (i > 0)
 			str_append(query, " AND");
 		str_printfa(query, " %s = ", sql_fields[i].name);
-		if (sql_dict_field_escape_value(query, dict, &sql_fields[i],
+		if (sql_dict_field_escape_value(query, dict, map, &sql_fields[i],
 						values[i], "", error_r) < 0)
 			return -1;
 	}
@@ -302,11 +304,11 @@ sql_dict_where_build(struct sql_dict *dict, const struct dict_sql_map *map,
 			str_append(query, " AND");
 		if (i < count2) {
 			str_printfa(query, " %s LIKE ", sql_fields[i].name);
-			if (sql_dict_field_escape_value(query, dict, &sql_fields[i],
+			if (sql_dict_field_escape_value(query, dict, map, &sql_fields[i],
 							values[i], "/%", error_r) < 0)
 				return -1;
 			str_printfa(query, " AND %s NOT LIKE ", sql_fields[i].name);
-			if (sql_dict_field_escape_value(query, dict, &sql_fields[i],
+			if (sql_dict_field_escape_value(query, dict, map, &sql_fields[i],
 							values[i], "/%/%", error_r) < 0)
 				return -1;
 		} else {
@@ -321,7 +323,7 @@ sql_dict_where_build(struct sql_dict *dict, const struct dict_sql_map *map,
 				str_append(query, " AND");
 			str_printfa(query, " %s LIKE ",
 				    sql_fields[i].name);
-			if (sql_dict_field_escape_value(query, dict, &sql_fields[i],
+			if (sql_dict_field_escape_value(query, dict, map, &sql_fields[i],
 							values[i], "/%", error_r) < 0)
 				return -1;
 		}
@@ -908,8 +910,9 @@ static int sql_dict_set_query(const struct dict_sql_build_query *build,
 		else {
 			enum dict_sql_type value_type =
 				sql_dict_map_type(fields[i].map);
-			if (sql_dict_value_escape(suffix, dict, value_type,
-				"value", fields[i].value, "", error_r) < 0)
+			if (sql_dict_value_escape(suffix, dict, fields[i].map,
+				value_type, "value", fields[i].value,
+				"", error_r) < 0)
 				return -1;
 		}
 	}
@@ -926,7 +929,7 @@ static int sql_dict_set_query(const struct dict_sql_build_query *build,
 	for (i = 0; i < count; i++) {
 		str_printfa(prefix, ",%s", sql_fields[i].name);
 		str_append_c(suffix, ',');
-		if (sql_dict_field_escape_value(suffix, dict, &sql_fields[i],
+		if (sql_dict_field_escape_value(suffix, dict, fields[0].map, &sql_fields[i],
 						extra_values[i], "", error_r) < 0)
 			return -1;
 	}
@@ -951,8 +954,9 @@ static int sql_dict_set_query(const struct dict_sql_build_query *build,
 		} else {
 			enum dict_sql_type value_type =
 				sql_dict_map_type(fields[i].map);
-			if (sql_dict_value_escape(prefix, dict, value_type,
-				"value", fields[i].value, "", error_r) < 0)
+			if (sql_dict_value_escape(prefix, dict, fields[i].map,
+				value_type, "value", fields[i].value,
+				"", error_r) < 0)
 				return -1;
 		}
 	}

From 5c44d09562ca322d9d25489ce65ac561b74bdca1 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 1 Jun 2016 12:37:22 +0300
Subject: [PATCH 243/450] master: Give a nicer error if unix/fifo_listener path
 is empty.

The previous error was simply:

unlink(/var/run/dovecot/) failed: Is a directory
---
 src/master/master-settings.c | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/src/master/master-settings.c b/src/master/master-settings.c
index 101222d9dde..da81dd075d3 100644
--- a/src/master/master-settings.c
+++ b/src/master/master-settings.c
@@ -277,21 +277,27 @@ expand_user(const char **user, enum service_user_default *default_r,
 	}
 }
 
-static void
+static bool
 fix_file_listener_paths(ARRAY_TYPE(file_listener_settings) *l,
 			pool_t pool, const struct master_settings *master_set,
-			ARRAY_TYPE(const_string) *all_listeners)
+			ARRAY_TYPE(const_string) *all_listeners,
+			const char **error_r)
 {
 	struct file_listener_settings *const *sets;
 	unsigned int base_dir_len = strlen(master_set->base_dir);
 	enum service_user_default user_default;
 
 	if (!array_is_created(l))
-		return;
+		return TRUE;
 
 	array_foreach(l, sets) {
 		struct file_listener_settings *set = *sets;
 
+		if (set->path[0] == '\0') {
+			*error_r = "path must not be empty";
+			return FALSE;
+		}
+
 		expand_user(&set->user, &user_default, master_set);
 		if (*set->path != '/') {
 			set->path = p_strconcat(pool, master_set->base_dir, "/",
@@ -305,6 +311,7 @@ fix_file_listener_paths(ARRAY_TYPE(file_listener_settings) *l,
 		if (set->mode != 0)
 			array_append(all_listeners, &set->path, 1);
 	}
+	return TRUE;
 }
 
 static void add_inet_listeners(ARRAY_TYPE(inet_listener_settings) *l,
@@ -570,10 +577,18 @@ master_settings_verify(void *_set, pool_t pool, const char **error_r)
 		    strcmp(service->name, "auth") == 0)
 			max_anvil_client_processes += process_limit;
 
-		fix_file_listener_paths(&service->unix_listeners,
-					pool, set, &all_listeners);
-		fix_file_listener_paths(&service->fifo_listeners,
-					pool, set, &all_listeners);
+		if (!fix_file_listener_paths(&service->unix_listeners, pool,
+					     set, &all_listeners, error_r)) {
+			*error_r = t_strdup_printf("service(%s): unix_listener: %s",
+						   service->name, *error_r);
+			return FALSE;
+		}
+		if (!fix_file_listener_paths(&service->fifo_listeners, pool,
+					     set, &all_listeners, error_r)) {
+			*error_r = t_strdup_printf("service(%s): fifo_listener: %s",
+						   service->name, *error_r);
+			return FALSE;
+		}
 		add_inet_listeners(&service->inet_listeners, &all_listeners);
 	}
 

From 8e0728383e141467168449e5517dd606d0b01eb9 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 30 May 2016 19:07:16 +0300
Subject: [PATCH 244/450] virtual: Moved virtual_mailbox_vfuncs to lib-storage.

This allows implementing other virtual storage backends.
---
 src/lib-storage/mail-storage-private.h | 22 +++++++++++++++++++++-
 src/plugins/fts/fts-api.c              | 12 +++++++++---
 src/plugins/fts/fts-search.c           | 23 ++++++++++-------------
 src/plugins/fts/fts-storage.c          |  7 ++-----
 src/plugins/virtual/virtual-storage.c  |  2 +-
 src/plugins/virtual/virtual-storage.h  | 20 --------------------
 6 files changed, 43 insertions(+), 43 deletions(-)

diff --git a/src/lib-storage/mail-storage-private.h b/src/lib-storage/mail-storage-private.h
index 6a2b53756e5..0aad8f83f80 100644
--- a/src/lib-storage/mail-storage-private.h
+++ b/src/lib-storage/mail-storage-private.h
@@ -150,6 +150,24 @@ struct mail_attachment_part {
 	const char *content_type, *content_disposition;
 };
 
+struct virtual_mailbox_vfuncs {
+	/* convert backend UIDs to virtual UIDs. if some backend UID doesn't
+	   exist in mailbox, it's simply ignored */
+	void (*get_virtual_uids)(struct mailbox *box,
+				 struct mailbox *backend_mailbox,
+				 const ARRAY_TYPE(seq_range) *backend_uids,
+				 ARRAY_TYPE(seq_range) *virtual_uids_r);
+	/* like get_virtual_uids(), but if a backend UID doesn't exist,
+	   convert it to 0. */
+	void (*get_virtual_uid_map)(struct mailbox *box,
+				    struct mailbox *backend_mailbox,
+				    const ARRAY_TYPE(seq_range) *backend_uids,
+				    ARRAY_TYPE(uint32_t) *virtual_uids_r);
+	void (*get_virtual_backend_boxes)(struct mailbox *box,
+					  ARRAY_TYPE(mailboxes) *mailboxes,
+					  bool only_with_msgs);
+};
+
 struct mailbox_vfuncs {
 	bool (*is_readonly)(struct mailbox *box);
 
@@ -286,7 +304,9 @@ struct mailbox {
 	struct mail_storage *storage;
 	struct mailbox_list *list;
 
-        struct mailbox_vfuncs v, *vlast;
+	struct mailbox_vfuncs v, *vlast;
+	/* virtual mailboxes: */
+	const struct virtual_mailbox_vfuncs *virtual_vfuncs;
 /* private: */
 	pool_t pool, metadata_pool;
 	/* Linked list of all mailboxes in this storage */
diff --git a/src/plugins/fts/fts-api.c b/src/plugins/fts/fts-api.c
index 66c09c900a7..f3a94223ce2 100644
--- a/src/plugins/fts/fts-api.c
+++ b/src/plugins/fts/fts-api.c
@@ -8,7 +8,6 @@
 #include "mail-storage-private.h"
 #include "mailbox-list-iter.h"
 #include "mail-search.h"
-#include "../virtual/virtual-storage.h"
 #include "fts-api-private.h"
 
 static ARRAY(const struct fts_backend *) backends;
@@ -90,7 +89,7 @@ int fts_backend_get_last_uid(struct fts_backend *backend, struct mailbox *box,
 {
 	struct fts_index_header hdr;
 
-	if (strcmp(box->storage->name, VIRTUAL_STORAGE_NAME) == 0) {
+	if (box->virtual_vfuncs != NULL) {
 		/* virtual mailboxes themselves don't have any indexes,
 		   so catch this call here */
 		if (!fts_index_get_header(box, &hdr))
@@ -228,7 +227,14 @@ int fts_backend_reset_last_uids(struct fts_backend *backend)
 
 int fts_backend_rescan(struct fts_backend *backend)
 {
-	if (strcmp(backend->ns->storage->name, VIRTUAL_STORAGE_NAME) == 0) {
+	struct mailbox *box;
+	bool virtual_storage;
+
+	box = mailbox_alloc(backend->ns->list, "", 0);
+	virtual_storage = box->virtual_vfuncs != NULL;
+	mailbox_free(&box);
+
+	if (virtual_storage) {
 		/* just reset the last-uids for a virtual storage. */
 		return fts_backend_reset_last_uids(backend);
 	}
diff --git a/src/plugins/fts/fts-search.c b/src/plugins/fts/fts-search.c
index ffbf3d4268b..acdad40dabc 100644
--- a/src/plugins/fts/fts-search.c
+++ b/src/plugins/fts/fts-search.c
@@ -5,7 +5,6 @@
 #include "str.h"
 #include "seq-range-array.h"
 #include "mail-search.h"
-#include "../virtual/virtual-storage.h"
 #include "fts-api-private.h"
 #include "fts-search-args.h"
 #include "fts-search-serialize.h"
@@ -63,7 +62,7 @@ static int fts_search_lookup_level_single(struct fts_search_context *fctx,
 }
 
 static void
-level_scores_add_vuids(struct virtual_mailbox *vbox,
+level_scores_add_vuids(struct mailbox *box,
 		       struct fts_search_level *level, struct fts_result *br)
 {
 	const struct fts_score_map *scores;
@@ -78,8 +77,8 @@ level_scores_add_vuids(struct virtual_mailbox *vbox,
 	t_array_init(&backend_uids, 64);
 	for (i = 0; i < count; i++)
 		seq_range_array_add(&backend_uids, scores[i].uid);
-	vbox->vfuncs.get_virtual_uid_map(&vbox->box, br->box,
-					 &backend_uids, &vuids_arr);
+	box->virtual_vfuncs->get_virtual_uid_map(box, br->box,
+						 &backend_uids, &vuids_arr);
 
 	i_assert(array_count(&vuids_arr) == array_count(&br->scores));
 	vuids = array_get(&vuids_arr, &count);
@@ -110,7 +109,6 @@ multi_add_lookup_result(struct fts_search_context *fctx,
 			struct mail_search_arg *args,
 			struct fts_multi_result *result)
 {
-	struct virtual_mailbox *vbox = (struct virtual_mailbox *)fctx->box;
 	ARRAY_TYPE(seq_range) vuids;
 	size_t orig_size;
 	unsigned int i;
@@ -132,21 +130,20 @@ multi_add_lookup_result(struct fts_search_context *fctx,
 
 		array_clear(&vuids);
 		if (array_is_created(&br->definite_uids)) {
-			vbox->vfuncs.get_virtual_uids(fctx->box, br->box,
-						      &br->definite_uids,
-						      &vuids);
+			fctx->box->virtual_vfuncs->get_virtual_uids(fctx->box,
+				br->box, &br->definite_uids, &vuids);
 		}
 		uid_range_to_seqs(fctx, &vuids, &level->definite_seqs);
 
 		array_clear(&vuids);
 		if (array_is_created(&br->maybe_uids)) {
-			vbox->vfuncs.get_virtual_uids(fctx->box, br->box,
-						      &br->maybe_uids, &vuids);
+			fctx->box->virtual_vfuncs->get_virtual_uids(fctx->box,
+				br->box, &br->maybe_uids, &vuids);
 		}
 		uid_range_to_seqs(fctx, &vuids, &level->maybe_seqs);
 
 		if (array_is_created(&br->scores))
-			level_scores_add_vuids(vbox, level, br);
+			level_scores_add_vuids(fctx->box, level, br);
 	}
 	return 0;
 }
@@ -157,7 +154,6 @@ static int fts_search_lookup_level_multi(struct fts_search_context *fctx,
 {
 	enum fts_lookup_flags flags = fctx->flags |
 		(and_args ? FTS_LOOKUP_FLAG_AND_ARGS : 0);
-	struct virtual_mailbox *vbox = (struct virtual_mailbox *)fctx->box;
 	ARRAY_TYPE(mailboxes) mailboxes_arr, tmp_mailboxes;
 	struct mailbox *const *mailboxes;
 	struct fts_backend *backend;
@@ -166,7 +162,8 @@ static int fts_search_lookup_level_multi(struct fts_search_context *fctx,
 	unsigned int i, j, mailbox_count;
 
 	p_array_init(&mailboxes_arr, fctx->result_pool, 8);
-	vbox->vfuncs.get_virtual_backend_boxes(fctx->box, &mailboxes_arr, TRUE);
+	fctx->box->virtual_vfuncs->get_virtual_backend_boxes(fctx->box,
+		&mailboxes_arr, TRUE);
 	array_sort(&mailboxes_arr, mailbox_cmp_fts_backend);
 
 	memset(&result, 0, sizeof(result));
diff --git a/src/plugins/fts/fts-storage.c b/src/plugins/fts/fts-storage.c
index e803921ff37..31b2fe93aec 100644
--- a/src/plugins/fts/fts-storage.c
+++ b/src/plugins/fts/fts-storage.c
@@ -9,7 +9,6 @@
 #include "mail-search-build.h"
 #include "mail-storage-private.h"
 #include "mailbox-list-private.h"
-#include "../virtual/virtual-storage.h"
 #include "fts-api-private.h"
 #include "fts-tokenizer.h"
 #include "fts-indexer.h"
@@ -208,8 +207,7 @@ fts_mailbox_search_init(struct mailbox_transaction_context *t,
 	fctx->args = args;
 	fctx->result_pool = pool_alloconly_create("fts results", 1024*64);
 	fctx->orig_matches = buffer_create_dynamic(default_pool, 64);
-	fctx->virtual_mailbox =
-		strcmp(t->box->storage->name, VIRTUAL_STORAGE_NAME) == 0;
+	fctx->virtual_mailbox = t->box->virtual_vfuncs != NULL;
 	fctx->enforced =
 		mail_user_plugin_getenv(t->box->storage->user,
 					"fts_enforced") != NULL;
@@ -534,8 +532,7 @@ void fts_mail_allocated(struct mail *_mail)
 	fmail = p_new(mail->pool, struct fts_mail, 1);
 	fmail->module_ctx.super = *v;
 	mail->vlast = &fmail->module_ctx.super;
-	fmail->virtual_mail =
-		strcmp(_mail->box->storage->name, VIRTUAL_STORAGE_NAME) == 0;
+	fmail->virtual_mail = _mail->box->virtual_vfuncs != NULL;
 
 	v->get_special = fts_mail_get_special;
 	v->precache = fts_mail_precache;
diff --git a/src/plugins/virtual/virtual-storage.c b/src/plugins/virtual/virtual-storage.c
index 908fe30f81f..5cb46c7add5 100644
--- a/src/plugins/virtual/virtual-storage.c
+++ b/src/plugins/virtual/virtual-storage.c
@@ -260,7 +260,7 @@ virtual_mailbox_alloc(struct mail_storage *_storage, struct mailbox_list *list,
 	mbox->box.storage = _storage;
 	mbox->box.list = list;
 	mbox->box.mail_vfuncs = &virtual_mail_vfuncs;
-	mbox->vfuncs = virtual_mailbox_vfuncs;
+	mbox->box.virtual_vfuncs = &virtual_mailbox_vfuncs;
 
 	index_storage_mailbox_alloc(&mbox->box, vname, flags, MAIL_INDEX_PREFIX);
 
diff --git a/src/plugins/virtual/virtual-storage.h b/src/plugins/virtual/virtual-storage.h
index 697a64c438e..c777091410f 100644
--- a/src/plugins/virtual/virtual-storage.h
+++ b/src/plugins/virtual/virtual-storage.h
@@ -114,24 +114,6 @@ struct virtual_backend_box {
 };
 ARRAY_DEFINE_TYPE(virtual_backend_box, struct virtual_backend_box *);
 
-struct virtual_mailbox_vfuncs {
-	/* convert backend UIDs to virtual UIDs. if some backend UID doesn't
-	   exist in mailbox, it's simply ignored */
-	void (*get_virtual_uids)(struct mailbox *box,
-				 struct mailbox *backend_mailbox,
-				 const ARRAY_TYPE(seq_range) *backend_uids,
-				 ARRAY_TYPE(seq_range) *virtual_uids_r);
-	/* like get_virtual_uids(), but if a backend UID doesn't exist,
-	   convert it to 0. */
-	void (*get_virtual_uid_map)(struct mailbox *box,
-				    struct mailbox *backend_mailbox,
-				    const ARRAY_TYPE(seq_range) *backend_uids,
-				    ARRAY_TYPE(uint32_t) *virtual_uids_r);
-	void (*get_virtual_backend_boxes)(struct mailbox *box,
-					  ARRAY_TYPE(mailboxes) *mailboxes,
-					  bool only_with_msgs);
-};
-
 struct virtual_mailbox {
 	struct mailbox box;
 	struct virtual_storage *storage;
@@ -161,8 +143,6 @@ struct virtual_mailbox {
 	ARRAY_TYPE(mailbox_virtual_patterns) list_include_patterns;
 	ARRAY_TYPE(mailbox_virtual_patterns) list_exclude_patterns;
 
-	struct virtual_mailbox_vfuncs vfuncs;
-
 	unsigned int uids_mapped:1;
 	unsigned int sync_initialized:1;
 	unsigned int inconsistent:1;

From fd3cdfde33d7bd01909de1754e0acfd6aa36c73c Mon Sep 17 00:00:00 2001
From: Baofeng Wang <baofeng.wang@dovecot.fi>
Date: Wed, 18 May 2016 14:42:21 +0300
Subject: [PATCH 245/450] lib-storage: add no_fts in struct mail_search_arg

When flag is set, FTS will not be performed.

fts plugin: handle no_fts flag

No fts will be performed once flag is set.
Original patch from Timo Sirainen <timo.sirainen@dovecot.fi>
---
 src/lib-storage/mail-search.h               | 1 +
 src/plugins/fts-lucene/lucene-wrapper.cc    | 6 ++++++
 src/plugins/fts-solr/fts-backend-solr-old.c | 2 ++
 src/plugins/fts-solr/fts-backend-solr.c     | 4 ++++
 src/plugins/fts/fts-api.c                   | 4 +++-
 src/plugins/fts/fts-storage.c               | 4 +++-
 6 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/lib-storage/mail-search.h b/src/lib-storage/mail-search.h
index d8447faebb6..e0a9fd0aa2d 100644
--- a/src/lib-storage/mail-search.h
+++ b/src/lib-storage/mail-search.h
@@ -101,6 +101,7 @@ struct mail_search_arg {
 	unsigned int match_always:1; /* result = 1 always */
 	unsigned int nonmatch_always:1; /* result = 0 always */
 	unsigned int fuzzy:1; /* use fuzzy matching for this arg */
+	unsigned int no_fts:1; /* do NOT call FTS */
 
 	int result; /* -1 = unknown, 0 = unmatched, 1 = matched */
 };
diff --git a/src/plugins/fts-lucene/lucene-wrapper.cc b/src/plugins/fts-lucene/lucene-wrapper.cc
index 89f99a3fcd9..138b70392ea 100644
--- a/src/plugins/fts-lucene/lucene-wrapper.cc
+++ b/src/plugins/fts-lucene/lucene-wrapper.cc
@@ -1200,6 +1200,9 @@ lucene_add_definite_query(struct lucene_index *index,
 	bool and_args = (flags & FTS_LOOKUP_FLAG_AND_ARGS) != 0;
 	Query *q;
 
+	if (arg->no_fts)
+		return false;
+
 	if (arg->match_not && !and_args) {
 		/* FIXME: we could handle this by doing multiple queries.. */
 		return false;
@@ -1266,6 +1269,9 @@ lucene_add_maybe_query(struct lucene_index *index,
 	bool and_args = (flags & FTS_LOOKUP_FLAG_AND_ARGS) != 0;
 	Query *q = NULL;
 
+	if (arg->no_fts)
+		return false;
+
 	if (arg->match_not) {
 		/* FIXME: we could handle this by doing multiple queries.. */
 		return false;
diff --git a/src/plugins/fts-solr/fts-backend-solr-old.c b/src/plugins/fts-solr/fts-backend-solr-old.c
index c3776441423..5ffbc8ad824 100644
--- a/src/plugins/fts-solr/fts-backend-solr-old.c
+++ b/src/plugins/fts-solr/fts-backend-solr-old.c
@@ -630,6 +630,8 @@ static int fts_backend_solr_optimize(struct fts_backend *backend ATTR_UNUSED)
 static bool
 solr_add_definite_query(string_t *str, struct mail_search_arg *arg)
 {
+	if (arg->no_fts)
+		return FALSE;
 	switch (arg->type) {
 	case SEARCH_TEXT: {
 		if (arg->match_not)
diff --git a/src/plugins/fts-solr/fts-backend-solr.c b/src/plugins/fts-solr/fts-backend-solr.c
index 3c2da2f73ab..5d0d3ac3902 100644
--- a/src/plugins/fts-solr/fts-backend-solr.c
+++ b/src/plugins/fts-solr/fts-backend-solr.c
@@ -664,6 +664,8 @@ static void solr_add_str_arg(string_t *str, struct mail_search_arg *arg)
 static bool
 solr_add_definite_query(string_t *str, struct mail_search_arg *arg)
 {
+	if (arg->no_fts)
+		return FALSE;
 	switch (arg->type) {
 	case SEARCH_TEXT: {
 		if (arg->match_not)
@@ -726,6 +728,8 @@ solr_add_definite_query_args(string_t *str, struct mail_search_arg *arg,
 static bool
 solr_add_maybe_query(string_t *str, struct mail_search_arg *arg)
 {
+	if (arg->no_fts)
+		return FALSE;
 	switch (arg->type) {
 	case SEARCH_HEADER:
 	case SEARCH_HEADER_ADDRESS:
diff --git a/src/plugins/fts/fts-api.c b/src/plugins/fts/fts-api.c
index f3a94223ce2..5eec770bc85 100644
--- a/src/plugins/fts/fts-api.c
+++ b/src/plugins/fts/fts-api.c
@@ -315,7 +315,9 @@ bool fts_backend_default_can_lookup(struct fts_backend *backend,
 		case SEARCH_HEADER_COMPRESS_LWSP:
 		case SEARCH_BODY:
 		case SEARCH_TEXT:
-			return TRUE;
+			if (!args->no_fts)
+				return TRUE;
+			break;
 		default:
 			break;
 		}
diff --git a/src/plugins/fts/fts-storage.c b/src/plugins/fts/fts-storage.c
index 31b2fe93aec..b4b500982d1 100644
--- a/src/plugins/fts/fts-storage.c
+++ b/src/plugins/fts/fts-storage.c
@@ -154,7 +154,9 @@ static bool fts_want_build_args(const struct mail_search_arg *args)
 			break;
 		case SEARCH_BODY:
 		case SEARCH_TEXT:
-			return TRUE;
+			if (!args->no_fts)
+				return TRUE;
+			break;
 		default:
 			break;
 		}

From 47cbd0ff754e5cf82800402acc34db1ef6abad48 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 3 Jun 2016 17:40:11 +0300
Subject: [PATCH 246/450] lib-index: Minor code cleanup - use better variable
 names

---
 src/lib-index/mail-index-sync-ext.c | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/src/lib-index/mail-index-sync-ext.c b/src/lib-index/mail-index-sync-ext.c
index 51d5853bba6..3abfd4bd9f4 100644
--- a/src/lib-index/mail-index-sync-ext.c
+++ b/src/lib-index/mail-index-sync-ext.c
@@ -262,30 +262,30 @@ sync_ext_resize(const struct mail_transaction_ext_intro *u,
 	struct mail_index_map *map = ctx->view->map;
 	struct mail_index_ext *ext;
 	struct mail_index_ext_header *ext_hdr;
-	uint32_t old_size, new_size, old_record_size;
+	uint32_t old_padded_hdr_size, new_padded_hdr_size, old_record_size;
 	bool modified = FALSE, reorder = FALSE;
 
 	ext = array_idx_modifiable(&map->extensions, ext_map_idx);
 
-	old_size = MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size);
-	new_size = MAIL_INDEX_HEADER_SIZE_ALIGN(u->hdr_size);
+	old_padded_hdr_size = MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size);
+	new_padded_hdr_size = MAIL_INDEX_HEADER_SIZE_ALIGN(u->hdr_size);
 
-	if (new_size < old_size) {
+	if (new_padded_hdr_size < old_padded_hdr_size) {
 		/* header shrank */
 		if (no_shrink)
-			new_size = old_size;
+			new_padded_hdr_size = old_padded_hdr_size;
 		else {
 			buffer_delete(map->hdr_copy_buf,
-				      ext->hdr_offset + new_size,
-				      old_size - new_size);
+				      ext->hdr_offset + new_padded_hdr_size,
+				      old_padded_hdr_size - new_padded_hdr_size);
 			ext->hdr_size = u->hdr_size;
 			modified = TRUE;
 		}
-	} else if (new_size > old_size) {
+	} else if (new_padded_hdr_size > old_padded_hdr_size) {
 		/* header grown */
 		buffer_insert_zero(map->hdr_copy_buf,
-				   ext->hdr_offset + old_size,
-				   new_size - old_size);
+				   ext->hdr_offset + old_padded_hdr_size,
+				   new_padded_hdr_size - old_padded_hdr_size);
 		ext->hdr_size = u->hdr_size;
 		modified = TRUE;
 	} else {
@@ -327,10 +327,11 @@ sync_ext_resize(const struct mail_transaction_ext_intro *u,
 		i_assert(map->hdr_base == map->hdr_copy_buf->data);
 	}
 
-	if (new_size != old_size) {
+	if (new_padded_hdr_size != old_padded_hdr_size) {
 		/* move all hdr_offset of all extensions after this one */
 		unsigned int i, count = array_count(&map->extensions);
-		ssize_t diff = (ssize_t)new_size - (ssize_t)old_size;
+		ssize_t diff = (ssize_t)new_padded_hdr_size -
+			(ssize_t)old_padded_hdr_size;
 
 		ext = array_idx_modifiable(&map->extensions, 0);
 		for (i = ext_map_idx + 1; i < count; i++) {

From f2c6d9c53d9408aa3fa3b07c215264ed4b11e33b Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 3 Jun 2016 17:52:39 +0300
Subject: [PATCH 247/450] lib-index: Fixed changing extension record sizes.

map needs to be cloned before any extension record size changes are done.
Otherwise the map cloning will crash or do something broken.
---
 src/lib-index/mail-index-sync-ext.c | 56 ++++++++++++++---------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/src/lib-index/mail-index-sync-ext.c b/src/lib-index/mail-index-sync-ext.c
index 3abfd4bd9f4..994a2331ac1 100644
--- a/src/lib-index/mail-index-sync-ext.c
+++ b/src/lib-index/mail-index-sync-ext.c
@@ -259,17 +259,31 @@ sync_ext_resize(const struct mail_transaction_ext_intro *u,
 		uint32_t ext_map_idx, struct mail_index_sync_map_ctx *ctx,
 		bool no_shrink)
 {
-	struct mail_index_map *map = ctx->view->map;
+	struct mail_index_map *map;
 	struct mail_index_ext *ext;
 	struct mail_index_ext_header *ext_hdr;
 	uint32_t old_padded_hdr_size, new_padded_hdr_size, old_record_size;
-	bool modified = FALSE, reorder = FALSE;
-
-	ext = array_idx_modifiable(&map->extensions, ext_map_idx);
+	bool reorder = FALSE;
 
+	ext = array_idx_modifiable(&ctx->view->map->extensions, ext_map_idx);
 	old_padded_hdr_size = MAIL_INDEX_HEADER_SIZE_ALIGN(ext->hdr_size);
 	new_padded_hdr_size = MAIL_INDEX_HEADER_SIZE_ALIGN(u->hdr_size);
 
+	if (ext->record_align != u->record_align ||
+	    ext->record_size != u->record_size) {
+		/* record changed */
+	} else if (new_padded_hdr_size < old_padded_hdr_size) {
+		/* header is shrunk. do we allow? */
+		if (no_shrink)
+			return;
+	} else if (ext->hdr_size == u->hdr_size) {
+		/* no changes */
+		return;
+	}
+	/* something changed. get ourself a new map before we start changing
+	   anything in it. */
+	map = mail_index_sync_get_atomic_map(ctx);
+
 	if (new_padded_hdr_size < old_padded_hdr_size) {
 		/* header shrank */
 		if (no_shrink)
@@ -279,7 +293,6 @@ sync_ext_resize(const struct mail_transaction_ext_intro *u,
 				      ext->hdr_offset + new_padded_hdr_size,
 				      old_padded_hdr_size - new_padded_hdr_size);
 			ext->hdr_size = u->hdr_size;
-			modified = TRUE;
 		}
 	} else if (new_padded_hdr_size > old_padded_hdr_size) {
 		/* header grown */
@@ -287,20 +300,17 @@ sync_ext_resize(const struct mail_transaction_ext_intro *u,
 				   ext->hdr_offset + old_padded_hdr_size,
 				   new_padded_hdr_size - old_padded_hdr_size);
 		ext->hdr_size = u->hdr_size;
-		modified = TRUE;
 	} else {
 		if (ext->hdr_size != u->hdr_size) {
 			/* aligned sizes were the same, but the actual sizes
 			   had changed */
 			ext->hdr_size = u->hdr_size;
-			modified = TRUE;
 		}
 	}
 
 	if (ext->record_align < u->record_align ||
 	    (ext->record_align > u->record_align && !no_shrink)) {
 		ext->record_align = u->record_align;
-		modified = TRUE;
 		reorder = TRUE;
 	}
 
@@ -308,24 +318,19 @@ sync_ext_resize(const struct mail_transaction_ext_intro *u,
 	if (ext->record_size < u->record_size ||
 	    (ext->record_size > u->record_size && !no_shrink)) {
 		ext->record_size = u->record_size;
-		modified = TRUE;
 		reorder = TRUE;
 	}
 
-	if (modified) {
-		i_assert((map->hdr_copy_buf->used % sizeof(uint64_t)) == 0);
-		map->hdr_base = map->hdr_copy_buf->data;
-		map->hdr.header_size = map->hdr_copy_buf->used;
+	i_assert((map->hdr_copy_buf->used % sizeof(uint64_t)) == 0);
+	map->hdr_base = map->hdr_copy_buf->data;
+	map->hdr.header_size = map->hdr_copy_buf->used;
 
-		ext_hdr = get_ext_header(map, ext);
-		ext_hdr->reset_id = ext->reset_id;
-		ext_hdr->hdr_size = ext->hdr_size;
-		ext_hdr->record_offset = ext->record_offset;
-		ext_hdr->record_size = ext->record_size;
-		ext_hdr->record_align = ext->record_align;
-	} else {
-		i_assert(map->hdr_base == map->hdr_copy_buf->data);
-	}
+	ext_hdr = get_ext_header(map, ext);
+	ext_hdr->reset_id = ext->reset_id;
+	ext_hdr->hdr_size = ext->hdr_size;
+	ext_hdr->record_offset = ext->record_offset;
+	ext_hdr->record_size = ext->record_size;
+	ext_hdr->record_align = ext->record_align;
 
 	if (new_padded_hdr_size != old_padded_hdr_size) {
 		/* move all hdr_offset of all extensions after this one */
@@ -340,13 +345,8 @@ sync_ext_resize(const struct mail_transaction_ext_intro *u,
 		}
 	}
 
-	if (reorder) {
-		map = mail_index_sync_get_atomic_map(ctx);
+	if (reorder)
 		sync_ext_reorder(map, ext_map_idx, old_record_size);
-	} else if (modified) {
-		/* header size changed. recreate index file. */
-		(void)mail_index_sync_get_atomic_map(ctx);
-	}
 }
 
 static bool

From e6696669bff159e93dc040f1c8574e938348353c Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 3 Jun 2016 17:54:36 +0300
Subject: [PATCH 248/450] lib-index: Allow growing ext record_size after
 mail_index_update_ext()

The existing records will just get some zero-padding at the end of records.
---
 src/lib-index/mail-index-transaction-update.c | 40 ++++++++++++++++---
 src/lib-index/mail-index-util.c               | 19 ++++++---
 src/lib-index/mail-index-util.h               |  2 +
 3 files changed, 50 insertions(+), 11 deletions(-)

diff --git a/src/lib-index/mail-index-transaction-update.c b/src/lib-index/mail-index-transaction-update.c
index ac1cf62bc5a..38f9905b97d 100644
--- a/src/lib-index/mail-index-transaction-update.c
+++ b/src/lib-index/mail-index-transaction-update.c
@@ -713,6 +713,33 @@ void mail_index_update_header(struct mail_index_transaction *t,
 	}
 }
 
+static void
+mail_index_ext_rec_updates_resize(struct mail_index_transaction *t,
+				  uint32_t ext_id, uint16_t new_record_size)
+{
+	ARRAY_TYPE(seq_array) *array, old_array;
+	unsigned int i;
+
+	if (!array_is_created(&t->ext_rec_updates))
+		return;
+	array = array_idx_modifiable(&t->ext_rec_updates, ext_id);
+	if (!array_is_created(array))
+		return;
+
+	old_array = *array;
+	memset(array, 0, sizeof(*array));
+	mail_index_seq_array_alloc(array, new_record_size);
+
+	/* copy the records' beginnings. leave the end zero-filled. */
+	for (i = 0; i < array_count(&old_array); i++) {
+		const void *old_record = array_idx(&old_array, i);
+
+		memcpy(array_append_space(array), old_record,
+		       old_array.arr.element_size);
+	}
+	array_free(&old_array);
+}
+
 void mail_index_ext_resize(struct mail_index_transaction *t, uint32_t ext_id,
 			   uint32_t hdr_size, uint16_t record_size,
 			   uint16_t record_align)
@@ -741,12 +768,13 @@ void mail_index_ext_resize(struct mail_index_transaction *t, uint32_t ext_id,
 		old_header_size = ext->hdr_size;
 	}
 
-	/* allow only header size changes if extension records have already
-	   been changed in transaction */
-	i_assert(!array_is_created(&t->ext_rec_updates) ||
-		 record_size == (uint16_t)-1 ||
-		 (old_record_size == record_size &&
-		  old_record_align == record_align));
+	if (record_size != old_record_size) {
+		/* if record_size grows, we'll just resize the existing
+		   ext_rec_updates array. it's not possible to shrink
+		   record_size without data loss. */
+		i_assert(record_size > old_record_size);
+		mail_index_ext_rec_updates_resize(t, ext_id, record_size);
+	}
 
 	t->log_ext_updates = TRUE;
 
diff --git a/src/lib-index/mail-index-util.c b/src/lib-index/mail-index-util.c
index b633539588f..659d622764d 100644
--- a/src/lib-index/mail-index-util.c
+++ b/src/lib-index/mail-index-util.c
@@ -116,6 +116,18 @@ bool mail_index_seq_array_lookup(const ARRAY_TYPE(seq_array) *array,
 					mail_index_seq_record_cmp, idx_r);
 }
 
+void mail_index_seq_array_alloc(ARRAY_TYPE(seq_array) *array,
+				size_t record_size)
+{
+	size_t aligned_record_size = (record_size + 3) & ~3;
+
+	i_assert(!array_is_created(array));
+
+	array_create(array, default_pool,
+		     sizeof(uint32_t) + aligned_record_size,
+		     1024 / (sizeof(uint32_t) + aligned_record_size));
+}
+
 bool mail_index_seq_array_add(ARRAY_TYPE(seq_array) *array, uint32_t seq,
 			      const void *record, size_t record_size,
 			      void *old_record)
@@ -126,11 +138,8 @@ bool mail_index_seq_array_add(ARRAY_TYPE(seq_array) *array, uint32_t seq,
 	/* records need to be 32bit aligned */
 	aligned_record_size = (record_size + 3) & ~3;
 
-	if (!array_is_created(array)) {
-		array_create(array, default_pool,
-			     sizeof(seq) + aligned_record_size,
-			     1024 / (sizeof(seq) + aligned_record_size));
-	}
+	if (!array_is_created(array))
+		mail_index_seq_array_alloc(array, record_size);
 	i_assert(array->arr.element_size == sizeof(seq) + aligned_record_size);
 
 	if (mail_index_seq_array_lookup(array, seq, &idx)) {
diff --git a/src/lib-index/mail-index-util.h b/src/lib-index/mail-index-util.h
index eeb17e94991..b61e16a23aa 100644
--- a/src/lib-index/mail-index-util.h
+++ b/src/lib-index/mail-index-util.h
@@ -13,6 +13,8 @@ int mail_index_unpack_num(const uint8_t **p, const uint8_t *end,
 
 bool mail_index_seq_array_lookup(const ARRAY_TYPE(seq_array) *array,
 				 uint32_t seq, unsigned int *idx_r);
+void mail_index_seq_array_alloc(ARRAY_TYPE(seq_array) *array,
+				size_t record_size);
 bool mail_index_seq_array_add(ARRAY_TYPE(seq_array) *array, uint32_t seq,
 			      const void *record, size_t record_size,
 			      void *old_record) ATTR_NULL(5);

From ec2ee6f35cb0734fc0aca3a906fb491d2b3b1ed3 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 3 Jun 2016 19:17:03 +0300
Subject: [PATCH 249/450] maildir: Avoid extra memory usage on duplicate
 uidlist entries

Also fixes assert-crashing in hash_table_insert() when it happens.
---
 src/lib-storage/index/maildir/maildir-uidlist.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/lib-storage/index/maildir/maildir-uidlist.c b/src/lib-storage/index/maildir/maildir-uidlist.c
index b0208f2672e..6208ed29cbe 100644
--- a/src/lib-storage/index/maildir/maildir-uidlist.c
+++ b/src/lib-storage/index/maildir/maildir-uidlist.c
@@ -1701,8 +1701,11 @@ maildir_uidlist_sync_next_partial(struct maildir_uidlist_sync_ctx *ctx,
 		rec = p_new(uidlist->record_pool,
 			    struct maildir_uidlist_rec, 1);
 		rec->uid = (uint32_t)-1;
+		rec->filename = p_strdup(uidlist->record_pool, filename);
 		array_append(&uidlist->records, &rec, 1);
 		uidlist->change_counter++;
+
+		hash_table_insert(uidlist->files, rec->filename, rec);
 	}
 	if (uid != 0) {
 		if (rec->uid != uid && rec->uid != (uint32_t)-1) {
@@ -1724,8 +1727,6 @@ maildir_uidlist_sync_next_partial(struct maildir_uidlist_sync_ctx *ctx,
 	rec->flags &= ~MAILDIR_UIDLIST_REC_FLAG_NEW_DIR;
 	rec->flags = (rec->flags | flags) &
 		~MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;
-	rec->filename = p_strdup(uidlist->record_pool, filename);
-	hash_table_insert(uidlist->files, rec->filename, rec);
 
 	ctx->finished = FALSE;
 	*rec_r = rec;
@@ -1816,6 +1817,8 @@ int maildir_uidlist_sync_next_uid(struct maildir_uidlist_sync_ctx *ctx,
 			/* didn't exist in uidlist, it's recent */
 			flags |= MAILDIR_UIDLIST_REC_FLAG_RECENT;
 		}
+		rec->filename = p_strdup(ctx->record_pool, filename);
+		hash_table_insert(ctx->files, rec->filename, rec);
 
 		array_append(&ctx->records, &rec, 1);
 	}
@@ -1826,8 +1829,6 @@ int maildir_uidlist_sync_next_uid(struct maildir_uidlist_sync_ctx *ctx,
 	}
 
 	rec->flags = (rec->flags | flags) & ~MAILDIR_UIDLIST_REC_FLAG_NONSYNCED;
-	rec->filename = p_strdup(ctx->record_pool, filename);
-	hash_table_insert(ctx->files, rec->filename, rec);
 	*rec_r = rec;
 	return 1;
 }

From e2ec3d69d910698f349dbb7c31de3eed2382aab5 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 3 Jun 2016 19:18:43 +0300
Subject: [PATCH 250/450] maildir: Detect duplicate keywords in
 dovecot-keywords file

Use the first such keyword's index, not the last.

Also fixes assert-crashing in hash_table_insert() when it happens.
---
 src/lib-storage/index/maildir/maildir-keywords.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/lib-storage/index/maildir/maildir-keywords.c b/src/lib-storage/index/maildir/maildir-keywords.c
index bc8d34a7386..161bd01286e 100644
--- a/src/lib-storage/index/maildir/maildir-keywords.c
+++ b/src/lib-storage/index/maildir/maildir-keywords.c
@@ -168,7 +168,8 @@ static int maildir_keywords_sync(struct maildir_keywords *mk)
 		*p++ = '\0';
 
 		if (str_to_uint(line, &idx) < 0 ||
-		    idx >= MAILDIR_MAX_KEYWORDS || *p == '\0') {
+		    idx >= MAILDIR_MAX_KEYWORDS || *p == '\0' ||
+		    hash_table_lookup(mk->hash, p) != NULL) {
 			/* shouldn't happen */
 			continue;
 		}

From 4313f95c69c7b59452d32818ea2bd12c126cb352 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 3 Jun 2016 19:20:35 +0300
Subject: [PATCH 251/450] lib-storage: Detect duplicate mailbox GUIDs in
 guid-cache.

Also fixes assert-crashing in hash_table_insert() when it happens.
---
 src/lib-storage/mailbox-guid-cache.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/lib-storage/mailbox-guid-cache.c b/src/lib-storage/mailbox-guid-cache.c
index f6ac6233df7..75488a18fb4 100644
--- a/src/lib-storage/mailbox-guid-cache.c
+++ b/src/lib-storage/mailbox-guid-cache.c
@@ -73,6 +73,11 @@ void mailbox_guid_cache_refresh(struct mailbox_list *list)
 			i_error("Couldn't get mailbox %s GUID: %s",
 				info->vname, mailbox_get_last_error(box, NULL));
 			list->guid_cache_errors = TRUE;
+		} else if ((rec = hash_table_lookup(list->guid_cache,
+				(const uint8_t *)metadata.guid)) != NULL) {
+			i_warning("Mailbox %s has duplicate GUID with %s: %s",
+				  info->vname, rec->vname,
+				  guid_128_to_string(metadata.guid));
 		} else {
 			rec = p_new(list->guid_cache_pool,
 				    struct mailbox_guid_cache_rec, 1);

From 8faaab53528cb7618846fe9e65b0db02efcd56c2 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 3 Jun 2016 19:58:29 +0300
Subject: [PATCH 252/450] lib-index: Fixed fsck handling extension with invalid
 header size

Fixes:

Panic: file mail-index-sync-ext.c: line 393 (mail_index_sync_ext_init_new): assertion failed: (hdr_buf->used == map->hdr.header_size)
---
 src/lib-index/mail-index-fsck.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/lib-index/mail-index-fsck.c b/src/lib-index/mail-index-fsck.c
index 850ae93c5d6..8b177885669 100644
--- a/src/lib-index/mail-index-fsck.c
+++ b/src/lib-index/mail-index-fsck.c
@@ -277,6 +277,7 @@ mail_index_fsck_extensions(struct mail_index *index, struct mail_index_map *map,
 					      "with invalid header size",
 					      i, name);
 			hdr->header_size = offset;
+			buffer_set_used_size(map->hdr_copy_buf, hdr->header_size);
 			break;
 		}
 		if (mail_index_map_ext_hdr_check(hdr, ext_hdr, name,
@@ -420,6 +421,7 @@ mail_index_fsck_map(struct mail_index *index, struct mail_index_map *map)
 	mail_index_fsck_records(index, map, &hdr);
 
 	map->hdr = hdr;
+	i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
 }
 
 int mail_index_fsck(struct mail_index *index)

From 8c8d70670208914dee3d0adff4b735e0e7da16c4 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 3 Jun 2016 20:00:14 +0300
Subject: [PATCH 253/450] lib-index: Added header-size asserts

---
 src/lib-index/mail-index-map-read.c      | 1 +
 src/lib-index/mail-index-modseq.c        | 1 +
 src/lib-index/mail-index-sync-ext.c      | 2 ++
 src/lib-index/mail-index-sync-keywords.c | 1 +
 src/lib-index/mail-index-sync-update.c   | 1 +
 5 files changed, 6 insertions(+)

diff --git a/src/lib-index/mail-index-map-read.c b/src/lib-index/mail-index-map-read.c
index 7081f8c0f36..e84c4b70c8d 100644
--- a/src/lib-index/mail-index-map-read.c
+++ b/src/lib-index/mail-index-map-read.c
@@ -238,6 +238,7 @@ mail_index_try_read_map(struct mail_index_map *map,
 
 	mail_index_map_copy_hdr(map, hdr);
 	map->hdr_base = map->hdr_copy_buf->data;
+	i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
 	return 1;
 }
 
diff --git a/src/lib-index/mail-index-modseq.c b/src/lib-index/mail-index-modseq.c
index 1d7c9ba3193..29ceb585d53 100644
--- a/src/lib-index/mail-index-modseq.c
+++ b/src/lib-index/mail-index-modseq.c
@@ -521,6 +521,7 @@ static void mail_index_modseq_update_header(struct mail_index_view *view,
 		buffer_write(map->hdr_copy_buf, ext->hdr_offset,
 			     &new_modseq_hdr, sizeof(new_modseq_hdr));
 		map->hdr_base = map->hdr_copy_buf->data;
+		i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
 	}
 }
 
diff --git a/src/lib-index/mail-index-sync-ext.c b/src/lib-index/mail-index-sync-ext.c
index 994a2331ac1..19beb8cc3f3 100644
--- a/src/lib-index/mail-index-sync-ext.c
+++ b/src/lib-index/mail-index-sync-ext.c
@@ -570,6 +570,7 @@ static void mail_index_sync_ext_clear(struct mail_index_view *view,
 	memset(buffer_get_space_unsafe(map->hdr_copy_buf, ext->hdr_offset,
 				       ext->hdr_size), 0, ext->hdr_size);
 	map->hdr_base = map->hdr_copy_buf->data;
+	i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
 
 	for (seq = 1; seq <= view->map->rec_map->records_count; seq++) {
 		rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq);
@@ -640,6 +641,7 @@ int mail_index_sync_ext_hdr_update(struct mail_index_sync_map_ctx *ctx,
 
 	buffer_write(map->hdr_copy_buf, ext->hdr_offset + offset, data, size);
 	map->hdr_base = map->hdr_copy_buf->data;
+	i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
 
 	if (ext->index_idx == ctx->view->index->modseq_ext_id)
 		mail_index_modseq_hdr_update(ctx->modseq_ctx);
diff --git a/src/lib-index/mail-index-sync-keywords.c b/src/lib-index/mail-index-sync-keywords.c
index 8546310038d..7d5d0964f69 100644
--- a/src/lib-index/mail-index-sync-keywords.c
+++ b/src/lib-index/mail-index-sync-keywords.c
@@ -190,6 +190,7 @@ keywords_header_add(struct mail_index_sync_map_ctx *ctx,
 
 	buffer_copy(map->hdr_copy_buf, ext->hdr_offset, buf, 0, buf->used);
 	map->hdr_base = map->hdr_copy_buf->data;
+	i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
 
 	if (mail_index_map_parse_keywords(map) < 0)
 		i_panic("Keyword update corrupted keywords header");
diff --git a/src/lib-index/mail-index-sync-update.c b/src/lib-index/mail-index-sync-update.c
index bb35a29fb69..0e0edee624f 100644
--- a/src/lib-index/mail-index-sync-update.c
+++ b/src/lib-index/mail-index-sync-update.c
@@ -496,6 +496,7 @@ static int sync_header_update(const struct mail_transaction_header_update *u,
 
 	buffer_write(map->hdr_copy_buf, u->offset, u + 1, u->size);
 	map->hdr_base = map->hdr_copy_buf->data;
+	i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
 
 	/* @UNSAFE */
 	if ((uint32_t)(u->offset + u->size) <= sizeof(map->hdr)) {

From a62105e9e4036858d717d080bca884a0e8ed90c5 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 3 Jun 2016 20:10:02 +0300
Subject: [PATCH 254/450] lib-index: Fixed extension resizing

Broken by 8483af4ff.
---
 src/lib-index/mail-index-sync-ext.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/lib-index/mail-index-sync-ext.c b/src/lib-index/mail-index-sync-ext.c
index 19beb8cc3f3..e758e53df30 100644
--- a/src/lib-index/mail-index-sync-ext.c
+++ b/src/lib-index/mail-index-sync-ext.c
@@ -283,6 +283,8 @@ sync_ext_resize(const struct mail_transaction_ext_intro *u,
 	/* something changed. get ourself a new map before we start changing
 	   anything in it. */
 	map = mail_index_sync_get_atomic_map(ctx);
+	/* ext was duplicated to the new map. */
+	ext = array_idx_modifiable(&map->extensions, ext_map_idx);
 
 	if (new_padded_hdr_size < old_padded_hdr_size) {
 		/* header shrank */

From dab6a443b64ebb41493ea13a833c1d81cc2cbad6 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 3 Jun 2016 20:14:01 +0300
Subject: [PATCH 255/450] lib-index: Fix duplicate fields in
 mail_cache_register_fields()

Broken by hash_table_insert() API change. The earlier code was also a bit
wrong by allocating a bit too much memory when there were duplicate fields
being registered.
---
 src/lib-index/mail-cache-fields.c | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/lib-index/mail-cache-fields.c b/src/lib-index/mail-cache-fields.c
index 98a030b9ca2..a46a2e400ea 100644
--- a/src/lib-index/mail-cache-fields.c
+++ b/src/lib-index/mail-cache-fields.c
@@ -105,7 +105,7 @@ void mail_cache_register_fields(struct mail_cache *cache,
 	char *name;
 	void *value;
 	unsigned int new_idx;
-	unsigned int i, j;
+	unsigned int i, j, registered_count;
 
 	new_idx = cache->fields_count;
 	for (i = 0; i < fields_count; i++) {
@@ -141,10 +141,11 @@ void mail_cache_register_fields(struct mail_cache *cache,
 			  cache->fields_count * sizeof(*cache->field_file_map),
 			  new_idx * sizeof(*cache->field_file_map));
 
+	registered_count = cache->fields_count;
 	for (i = 0; i < fields_count; i++) {
 		unsigned int idx = fields[i].idx;
 
-		if (idx < cache->fields_count)
+		if (idx < registered_count)
 			continue;
 
 		/* new index - save it */
@@ -159,7 +160,9 @@ void mail_cache_register_fields(struct mail_cache *cache,
 
 		hash_table_insert(cache->field_name_hash, name,
 				  POINTER_CAST(idx));
+		registered_count++;
 	}
+	i_assert(registered_count == new_idx);
 	cache->fields_count = new_idx;
 }
 

From 3a398ecbc1ce74dcb42667424b476ce4b6012831 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sat, 4 Jun 2016 03:47:37 +0300
Subject: [PATCH 256/450] lazy-expunge: Fixed crash on error handling

---
 src/plugins/lazy-expunge/lazy-expunge-plugin.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plugins/lazy-expunge/lazy-expunge-plugin.c b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
index baa6ffa13be..0a92de6090d 100644
--- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c
+++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
@@ -372,7 +372,7 @@ lazy_expunge_transaction_commit(struct mailbox_transaction_context *ctx,
 
 	if (lt->dest_trans != NULL && lt->delayed_error == MAIL_ERROR_NONE) {
 		if (mailbox_transaction_commit(&lt->dest_trans) < 0) {
-			lazy_expunge_set_error(lt, lt->dest_trans->box->storage);
+			lazy_expunge_set_error(lt, ctx->box->storage);
 		}
 	}
 

From 9f8c464a886ca45e20dec67c3ef3eefd663da682 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 5 Jun 2016 04:29:00 +0300
Subject: [PATCH 257/450] lib-dcrypt: Fixed function return type.

---
 src/lib-dcrypt/dcrypt-openssl.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index b8578769b46..24dff266cd2 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -1223,7 +1223,7 @@ int dcrypt_openssl_load_public_key_dovecot_v1(struct dcrypt_public_key **key_r,
 }
 
 static
-bool dcrypt_openssl_load_public_key_dovecot_v2(struct dcrypt_public_key **key_r,
+int dcrypt_openssl_load_public_key_dovecot_v2(struct dcrypt_public_key **key_r,
 	int len, const char **input, const char **error_r)
 {
 	if (len != 2 || strlen(input[1]) < 2 || (strlen(input[1])%2) != 0) {

From 25349a4d8de0f7345e9f76fe7be7d79eda5fd73d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 5 Jun 2016 14:54:05 +0300
Subject: [PATCH 258/450] director: Fixed ignoring an obsolete up/down change
 while host is desynced.

---
 src/director/director-connection.c | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/director/director-connection.c b/src/director/director-connection.c
index 2beb3270735..1ad6e02f422 100644
--- a/src/director/director-connection.c
+++ b/src/director/director-connection.c
@@ -927,6 +927,11 @@ director_cmd_host_int(struct director_connection *conn, const char *const *args,
 
 			str_printfa(str, "director(%s): Host %s is being updated before previous update had finished (",
 				  conn->name, net_ip2addr(&host->ip));
+			if (host->down != down &&
+			    host->last_updown_change > last_updown_change) {
+				/* our host has a newer change. preserve it. */
+				down = host->down;
+			}
 			if (host->down != down) {
 				if (host->down)
 					str_append(str, "down -> up");
@@ -942,10 +947,6 @@ director_cmd_host_int(struct director_connection *conn, const char *const *args,
 			str_append(str, ") - ");
 
 			vhost_count = I_MIN(vhost_count, host->vhost_count);
-			if (host->down != down) {
-				if (host->last_updown_change <= last_updown_change)
-					down = host->last_updown_change;
-			}
 			last_updown_change = I_MAX(last_updown_change,
 						   host->last_updown_change);
 			str_printfa(str, "setting to state=%s vhosts=%u",

From 9cc085fa1795e57223971486cd79bd13071c0a06 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 5 Jun 2016 15:11:56 +0300
Subject: [PATCH 259/450] auth: Fixed error handling in passdb/userdb dict
 config parsing

---
 src/auth/db-dict.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/auth/db-dict.c b/src/auth/db-dict.c
index 50f9f306d49..6720de150d8 100644
--- a/src/auth/db-dict.c
+++ b/src/auth/db-dict.c
@@ -193,8 +193,9 @@ static bool parse_section(const char *type, const char *name,
 				ctx->cur_key->parsed_format =
 					DB_DICT_VALUE_FORMAT_JSON;
 			} else {
-				return t_strconcat("Unknown key format: ",
-						   ctx->cur_key->format, NULL);
+				*errormsg = t_strconcat("Unknown key format: ",
+					ctx->cur_key->format, NULL);
+				return FALSE;
 			}
 		}
 		ctx->cur_key = NULL;

From 6c65c2f0d4c7e7fa1db6d8496fa82cd6cff6be03 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 5 Jun 2016 15:18:04 +0300
Subject: [PATCH 260/450] mdbox: Fix want_altpath flags/boolean mixup

There was only a single flag, so this wasn't actually currently broken.
---
 src/lib-storage/index/dbox-multi/mdbox-map.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-storage/index/dbox-multi/mdbox-map.c b/src/lib-storage/index/dbox-multi/mdbox-map.c
index fce5936d05d..5bc6548bf7c 100644
--- a/src/lib-storage/index/dbox-multi/mdbox-map.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-map.c
@@ -1086,7 +1086,7 @@ int mdbox_map_append_next(struct mdbox_map_append_context *ctx,
 		ret = 1;
 		existing = TRUE;
 	} else {
-		ret = mdbox_map_find_appendable_file(ctx, mail_size, flags,
+		ret = mdbox_map_find_appendable_file(ctx, mail_size, want_altpath,
 						     &file_append, output_r);
 		existing = FALSE;
 	}

From c1e4cd378d8f6bbe8dbbd78d81dbe6c97a059aea Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 5 Jun 2016 15:23:45 +0300
Subject: [PATCH 261/450] mbox: Fixed expunging first mail with CRLF linefeeds.

---
 src/lib-storage/index/mbox/mbox-sync-rewrite.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-storage/index/mbox/mbox-sync-rewrite.c b/src/lib-storage/index/mbox/mbox-sync-rewrite.c
index ef902441a7d..63bb4c125cc 100644
--- a/src/lib-storage/index/mbox/mbox-sync-rewrite.c
+++ b/src/lib-storage/index/mbox/mbox-sync-rewrite.c
@@ -354,7 +354,7 @@ static int mbox_sync_read_next(struct mbox_sync_context *sync_ctx,
 	}
 
 	first_mail_expunge_extra = 1 +
-		sync_ctx->first_mail_crlf_expunged ? 1 : 0;
+		(sync_ctx->first_mail_crlf_expunged ? 1 : 0);
 	if (mails[idx].from_offset +
 	    first_mail_expunge_extra - expunged_space != 0) {
 		sync_ctx->dest_first_mail = mails[idx].from_offset == 0;

From 39918071d9b8bb639a0e17ddfbde66a0611dbbe8 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 5 Jun 2016 15:26:11 +0300
Subject: [PATCH 262/450] lib-storage: Search args equalness checks didn't
 compare keywords correctly.

This may have broken the search in some situations.
---
 src/lib-storage/mail-search.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-storage/mail-search.c b/src/lib-storage/mail-search.c
index 0deaf397e51..4ffbba09b65 100644
--- a/src/lib-storage/mail-search.c
+++ b/src/lib-storage/mail-search.c
@@ -609,7 +609,7 @@ bool mail_search_arg_one_equals(const struct mail_search_arg *arg1,
 	case SEARCH_FLAGS:
 		return arg1->value.flags == arg2->value.flags;
 	case SEARCH_KEYWORDS:
-		return strcasecmp(arg1->value.str, arg2->value.str);
+		return strcasecmp(arg1->value.str, arg2->value.str) == 0;
 
 	case SEARCH_BEFORE:
 	case SEARCH_ON:

From e6447acb90fe93a3dfbb35ad44e1a957126ed544 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 5 Jun 2016 15:31:20 +0300
Subject: [PATCH 263/450] mailbox-alias plugin: Fixed error handling

---
 src/plugins/mailbox-alias/mailbox-alias-plugin.c | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/plugins/mailbox-alias/mailbox-alias-plugin.c b/src/plugins/mailbox-alias/mailbox-alias-plugin.c
index 6cfe6a801f0..d7ee7f9deb5 100644
--- a/src/plugins/mailbox-alias/mailbox-alias-plugin.c
+++ b/src/plugins/mailbox-alias/mailbox-alias-plugin.c
@@ -225,7 +225,9 @@ static int mailbox_alias_delete(struct mailbox *box)
 		return -1;
 	}
 
-	if (mailbox_is_alias_symlink(box)) {
+	if ((ret = mailbox_is_alias_symlink(box)) < 0)
+		return -1;
+	if (ret > 0) {
 		/* we're deleting an alias mailbox. we'll need to handle this
 		   explicitly since box->name points to the original mailbox */
 		symlink_name = alist->module_ctx.super.
@@ -245,12 +247,16 @@ static int mailbox_alias_rename(struct mailbox *src, struct mailbox *dest)
 	struct mailbox_alias_mailbox *abox = MAILBOX_ALIAS_CONTEXT(src);
 	int ret;
 
-	if (mailbox_is_alias_symlink(src)) {
+	if ((ret = mailbox_is_alias_symlink(src)) < 0)
+		return -1;
+	else if (ret > 0) {
 		mail_storage_set_error(src->storage, MAIL_ERROR_NOTPOSSIBLE,
 				       "Can't rename alias mailboxes");
 		return -1;
 	}
-	if (mailbox_is_alias_symlink(dest)) {
+	if ((ret = mailbox_is_alias_symlink(dest)) < 0)
+		return -1;
+	else {
 		mail_storage_set_error(src->storage, MAIL_ERROR_NOTPOSSIBLE,
 				       "Can't rename to mailbox alias");
 		return -1;

From 8035f936c3805046b4ca4069e967189ca926e84a Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 5 Jun 2016 15:32:09 +0300
Subject: [PATCH 264/450] fts-lucene: Fixed error handling when checking if
 settings had changed.

---
 src/plugins/fts-lucene/fts-backend-lucene.c | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/plugins/fts-lucene/fts-backend-lucene.c b/src/plugins/fts-lucene/fts-backend-lucene.c
index f098e481b18..3420debe2d6 100644
--- a/src/plugins/fts-lucene/fts-backend-lucene.c
+++ b/src/plugins/fts-lucene/fts-backend-lucene.c
@@ -178,11 +178,15 @@ fts_backend_lucene_get_last_uid(struct fts_backend *_backend,
 		FTS_LUCENE_USER_CONTEXT(_backend->ns->user);
 	struct fts_index_header hdr;
 	uint32_t set_checksum;
+	int ret;
 
 	if (fts_index_get_header(box, &hdr)) {
 		set_checksum = fts_lucene_settings_checksum(&fuser->set);
-		if (!fts_index_have_compatible_settings(_backend->ns->list,
-							set_checksum)) {
+		ret = fts_index_have_compatible_settings(_backend->ns->list,
+							 set_checksum);
+		if (ret < 0)
+			return -1;
+		if (ret == 0) {
 			/* need to rebuild the index */
 			*last_uid_r = 0;
 		} else {

From ddc96f79d209173926141eed955da55c8dfa298a Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 5 Jun 2016 15:35:13 +0300
Subject: [PATCH 265/450] lib-storage: Fixed error handling in list=children
 checking

---
 src/lib-storage/list/mailbox-list-iter.c | 29 ++++++++++++++----------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/src/lib-storage/list/mailbox-list-iter.c b/src/lib-storage/list/mailbox-list-iter.c
index bb8de56f5df..2c1381b834c 100644
--- a/src/lib-storage/list/mailbox-list-iter.c
+++ b/src/lib-storage/list/mailbox-list-iter.c
@@ -354,33 +354,38 @@ mailbox_list_ns_prefix_match(struct ns_list_iterate_context *ctx,
 	return ret;
 }
 
-static bool
+static int
 ns_prefix_is_visible(struct ns_list_iterate_context *ctx,
 		     struct mail_namespace *ns)
 {
+	int ret;
+
 	if ((ns->flags & NAMESPACE_FLAG_LIST_PREFIX) != 0)
-		return TRUE;
+		return 1;
 	if ((ns->flags & NAMESPACE_FLAG_LIST_CHILDREN) != 0) {
-		if (mailbox_list_match_anything(ctx, ns, ns->prefix))
-			return TRUE;
+		if ((ret = mailbox_list_match_anything(ctx, ns, ns->prefix)) != 0)
+			return ret;
 	}
-	return FALSE;
+	return 0;
 }
 
-static bool
+static int
 ns_prefix_has_visible_child_namespace(struct ns_list_iterate_context *ctx,
 				      const char *prefix)
 {
 	struct mail_namespace *ns;
 	unsigned int prefix_len = strlen(prefix);
+	int ret;
 
 	for (ns = ctx->namespaces; ns != NULL; ns = ns->next) {
 		if (ns->prefix_len > prefix_len &&
-		    strncmp(ns->prefix, prefix, prefix_len) == 0 &&
-		    ns_prefix_is_visible(ctx, ns))
-			return TRUE;
+		    strncmp(ns->prefix, prefix, prefix_len) == 0) {
+			ret = ns_prefix_is_visible(ctx, ns);
+			if (ret != 0)
+				return ret;
+		}
 	}
-	return FALSE;
+	return 0;
 }
 
 static bool
@@ -410,8 +415,8 @@ mailbox_list_match_anything(struct ns_list_iterate_context *ctx,
 	const char *pattern;
 	int ret;
 
-	if (ns_prefix_has_visible_child_namespace(ctx, prefix))
-		return 1;
+	if ((ret = ns_prefix_has_visible_child_namespace(ctx, prefix)) != 0)
+		return ret;
 
 	pattern = t_strconcat(prefix, "%", NULL);
 	list_iter = mailbox_list_iter_init(ns->list, pattern, list_flags);

From df0fe3927a8fb052f138a3d053c6d3865f23d677 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 5 Jun 2016 15:57:48 +0300
Subject: [PATCH 266/450] lib-dcrypt: Fixed error handling in
 dcrypt_key_id_public()

---
 src/lib-dcrypt/dcrypt-openssl.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index 24dff266cd2..797214026f2 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -1920,7 +1920,7 @@ bool dcrypt_openssl_public_key_id(struct dcrypt_public_key *key, const char *alg
 	unsigned char buf[EVP_MD_size(md)];
 	EVP_PKEY *pub = (EVP_PKEY*)key;
 	const char *ptr;
-	int ec;
+	bool res;
 	if (pub == NULL) {
 		if (error_r != NULL)
 			*error_r = "key is NULL";
@@ -1945,10 +1945,10 @@ bool dcrypt_openssl_public_key_id(struct dcrypt_public_key *key, const char *alg
 	if (EVP_DigestInit_ex(ctx, md, NULL) < 1 ||
 	    EVP_DigestUpdate(ctx, (const unsigned char*)ptr, len) < 1 ||
 	    EVP_DigestFinal_ex(ctx, buf, &hlen) < 1) {
-		ec = dcrypt_openssl_error(error_r);
+		res = dcrypt_openssl_error(error_r);
 	} else {
 		buffer_append(result, buf, hlen);
-		ec = 0;
+		res = TRUE;
 	}
 
 #if SSLEAY_VERSION_NUMBER >= 0x1010000fL
@@ -1958,7 +1958,7 @@ bool dcrypt_openssl_public_key_id(struct dcrypt_public_key *key, const char *alg
 #endif
 	BIO_vfree(b);
 
-	return ec == 0;
+	return res;
 }
 
 static struct dcrypt_vfs dcrypt_openssl_vfs = {

From 3920a5f312900c0681e7c3f01ce88b62143b566d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 5 Jun 2016 16:40:27 +0300
Subject: [PATCH 267/450] director: Fixed error handling when directors support
 incompatible tags

Connection should have been disconnected immediately, not after the next
command that would have produced "Incompatible protocol".
---
 src/director/director-connection.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/director/director-connection.c b/src/director/director-connection.c
index 1ad6e02f422..d8b58d3c563 100644
--- a/src/director/director-connection.c
+++ b/src/director/director-connection.c
@@ -1239,7 +1239,7 @@ director_connection_handle_handshake(struct director_connection *conn,
 		if (conn->minor_version < DIRECTOR_VERSION_TAGS_V2 &&
 		    mail_hosts_have_tags(conn->dir->mail_hosts)) {
 			i_error("director(%s): Director version supports incompatible tags", conn->name);
-			return FALSE;
+			return -1;
 		}
 		conn->version_received = TRUE;
 		if (conn->done_pending) {

From ff9e37db1f1ad741dee04e0b6c14106d5fb017b5 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 5 Jun 2016 16:51:12 +0300
Subject: [PATCH 268/450] lib-mail: Added comment about invalid timezones in
 message_date_parse()

---
 src/lib-mail/message-date.c | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/lib-mail/message-date.c b/src/lib-mail/message-date.c
index 49fedd8a545..ab52e783277 100644
--- a/src/lib-mail/message-date.c
+++ b/src/lib-mail/message-date.c
@@ -222,7 +222,11 @@ message_date_parser_tokens(struct message_date_parser_context *ctx,
 		/* missing timezone */
 		*timezone_offset_r = 0;
 	} else {
-		/* timezone */
+		/* timezone. invalid timezones are treated as GMT, because
+		   we may not know all the possible timezones that are used
+		   and it's better to give at least a mostly correct reply.
+		   FIXME: perhaps some different strict version of this
+		   function would be useful? */
 		*timezone_offset_r = parse_timezone(value, len);
 	}
 

From be08677cd3bb1e05804c5ec29691c902e1fd4386 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 6 Jun 2016 00:07:53 +0300
Subject: [PATCH 269/450] lib-index: Extension record size resizing was still
 broken.

Fixes assert-crash:
Panic: file mail-index-util.c: line 143 (mail_index_seq_array_add): assertion failed: (array->arr.element_size == sizeof(seq) + aligned_record_size)
---
 src/lib-index/mail-index-transaction-update.c | 27 +++++++++++++++----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/src/lib-index/mail-index-transaction-update.c b/src/lib-index/mail-index-transaction-update.c
index 38f9905b97d..dff497e358f 100644
--- a/src/lib-index/mail-index-transaction-update.c
+++ b/src/lib-index/mail-index-transaction-update.c
@@ -744,18 +744,19 @@ void mail_index_ext_resize(struct mail_index_transaction *t, uint32_t ext_id,
 			   uint32_t hdr_size, uint16_t record_size,
 			   uint16_t record_align)
 {
+	const struct mail_index_registered_ext *rext;
+	const struct mail_transaction_ext_intro *resizes;
+	unsigned int resizes_count;
 	struct mail_transaction_ext_intro intro;
-	uint32_t old_record_size, old_record_align, old_header_size;
+	uint32_t old_record_size = 0, old_record_align, old_header_size;
 
 	memset(&intro, 0, sizeof(intro));
+	rext = array_idx(&t->view->index->extensions, ext_id);
 
 	/* get ext_id from transaction's map if it's there */
 	if (!mail_index_map_get_ext_idx(t->view->map, ext_id, &intro.ext_id)) {
 		/* have to create it */
-		const struct mail_index_registered_ext *rext;
-
 		intro.ext_id = (uint32_t)-1;
-		rext = array_idx(&t->view->index->extensions, ext_id);
 		old_record_size = rext->record_size;
 		old_record_align = rext->record_align;
 		old_header_size = rext->hdr_size;
@@ -763,11 +764,27 @@ void mail_index_ext_resize(struct mail_index_transaction *t, uint32_t ext_id,
 		const struct mail_index_ext *ext;
 
 		ext = array_idx(&t->view->map->extensions, intro.ext_id);
-		old_record_size = ext->record_size;
 		old_record_align = ext->record_align;
 		old_header_size = ext->hdr_size;
 	}
 
+	/* get the record size. if there are any existing record updates,
+	   they're using the registered size, not the map's existing
+	   record_size. */
+	if (array_is_created(&t->ext_resizes))
+		resizes = array_get(&t->ext_resizes, &resizes_count);
+	else {
+		resizes = NULL;
+		resizes_count = 0;
+	}
+	if (ext_id < resizes_count && resizes[ext_id].name_size != 0) {
+		/* already resized once. use the resized value. */
+		old_record_size = resizes[ext_id].record_size;
+	} else {
+		/* use the registered values. */
+		record_size = rext->record_size;
+	}
+
 	if (record_size != old_record_size) {
 		/* if record_size grows, we'll just resize the existing
 		   ext_rec_updates array. it's not possible to shrink

From 507292b595849b0516ba9602d3b2880e61de8332 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 6 Jun 2016 02:26:09 +0300
Subject: [PATCH 270/450] dict: Fixed hang when pipelining multiple commands.

---
 src/dict/dict-connection.c | 41 ++++++++++++++++++++++----------------
 1 file changed, 24 insertions(+), 17 deletions(-)

diff --git a/src/dict/dict-connection.c b/src/dict/dict-connection.c
index 3040ec3132e..7484baa8012 100644
--- a/src/dict/dict-connection.c
+++ b/src/dict/dict-connection.c
@@ -104,11 +104,32 @@ static int dict_connection_dict_init(struct dict_connection *conn)
 	return 0;
 }
 
-static void dict_connection_input(struct dict_connection *conn)
+static void dict_connection_input_more(struct dict_connection *conn)
 {
 	const char *line;
 	int ret;
 
+	while ((line = i_stream_next_line(conn->input)) != NULL) {
+		T_BEGIN {
+			ret = dict_command_input(conn, line);
+		} T_END;
+		if (ret < 0) {
+			dict_connection_destroy(conn);
+			break;
+		}
+		if (array_count(&conn->cmds) >= DICT_CONN_MAX_PENDING_COMMANDS) {
+			io_remove(&conn->io);
+			if (conn->to_input != NULL)
+				timeout_remove(&conn->to_input);
+			break;
+		}
+	}
+}
+
+static void dict_connection_input(struct dict_connection *conn)
+{
+	const char *line;
+
 	if (conn->to_input != NULL)
 		timeout_remove(&conn->to_input);
 
@@ -143,21 +164,7 @@ static void dict_connection_input(struct dict_connection *conn)
 		}
 	}
 
-	while ((line = i_stream_next_line(conn->input)) != NULL) {
-		T_BEGIN {
-			ret = dict_command_input(conn, line);
-		} T_END;
-		if (ret < 0) {
-			dict_connection_destroy(conn);
-			break;
-		}
-		if (array_count(&conn->cmds) >= DICT_CONN_MAX_PENDING_COMMANDS) {
-			io_remove(&conn->io);
-			if (conn->to_input != NULL)
-				timeout_remove(&conn->to_input);
-			break;
-		}
-	}
+	dict_connection_input_more(conn);
 }
 
 void dict_connection_continue_input(struct dict_connection *conn)
@@ -167,7 +174,7 @@ void dict_connection_continue_input(struct dict_connection *conn)
 
 	conn->io = io_add(conn->fd, IO_READ, dict_connection_input, conn);
 	if (conn->to_input == NULL)
-		conn->to_input = timeout_add_short(0, dict_connection_input, conn);
+		conn->to_input = timeout_add_short(0, dict_connection_input_more, conn);
 }
 
 static int dict_connection_output(struct dict_connection *conn)

From 3eb750330f8bad57d00a75debf910432f3957f9c Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 6 Jun 2016 02:27:35 +0300
Subject: [PATCH 271/450] fs-randomfail: Support failures after asynchronous
 commands have already finished.

---
 src/lib-fs/fs-randomfail.c | 94 +++++++++++++++++++++++++++-----------
 1 file changed, 67 insertions(+), 27 deletions(-)

diff --git a/src/lib-fs/fs-randomfail.c b/src/lib-fs/fs-randomfail.c
index 4cff6671f1c..3a397dec996 100644
--- a/src/lib-fs/fs-randomfail.c
+++ b/src/lib-fs/fs-randomfail.c
@@ -27,6 +27,7 @@ struct randomfail_fs_file {
 	struct fs_file file;
 	struct fs_file *super, *super_read;
 	struct istream *input;
+	bool op_pending[FS_OP_COUNT];
 
 	struct ostream *super_output;
 };
@@ -254,25 +255,48 @@ fs_randomfail_set_async_callback(struct fs_file *_file,
 	fs_file_set_async_callback(file->super, callback, context);
 }
 
-static bool fs_random_fail(struct fs *_fs, enum fs_op op)
+static bool fs_random_fail(struct fs *_fs, int divider, enum fs_op op)
 {
 	struct randomfail_fs *fs = (struct randomfail_fs *)_fs;
 
 	if (fs->op_probability[op] == 0)
 		return FALSE;
-	if ((unsigned int)(rand() % 100) <= fs->op_probability[op]) {
+	if ((unsigned int)(rand() % (100*divider)) <= fs->op_probability[op]) {
 		fs_set_error(_fs, RANDOMFAIL_ERROR);
 		return TRUE;
 	}
 	return FALSE;
 }
 
+static bool
+fs_file_random_fail_begin(struct randomfail_fs_file *file, enum fs_op op)
+{
+	if (!file->op_pending[op]) {
+		if (fs_random_fail(file->file.fs, 2, op))
+			return TRUE;
+	}
+	file->op_pending[op] = TRUE;
+	return FALSE;
+}
+
+static int
+fs_file_random_fail_end(struct randomfail_fs_file *file,
+			int ret, enum fs_op op)
+{
+	if (ret == 0 || errno != ENOENT) {
+		if (fs_random_fail(file->file.fs, 2, op))
+			return TRUE;
+		file->op_pending[op] = FALSE;
+	}
+	return ret;
+}
+
 static bool
 fs_random_fail_range(struct fs *_fs, enum fs_op op, uoff_t *offset_r)
 {
 	struct randomfail_fs *fs = (struct randomfail_fs *)_fs;
 
-	if (!fs_random_fail(_fs, op))
+	if (!fs_random_fail(_fs, 1, op))
 		return FALSE;
 	*offset_r = fs->range_start[op] +
 		rand() % (fs->range_end[op] - fs->range_start[op] + 1);
@@ -281,7 +305,7 @@ fs_random_fail_range(struct fs *_fs, enum fs_op op, uoff_t *offset_r)
 
 static int fs_randomfail_wait_async(struct fs *_fs)
 {
-	if (fs_random_fail(_fs, FS_OP_WAIT))
+	if (fs_random_fail(_fs, 1, FS_OP_WAIT))
 		return -1;
 	return fs_wait_async(_fs->parent);
 }
@@ -300,17 +324,19 @@ fs_randomfail_get_metadata(struct fs_file *_file,
 			   const ARRAY_TYPE(fs_metadata) **metadata_r)
 {
 	struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file;
+	int ret;
 
-	if (fs_random_fail(_file->fs, FS_OP_METADATA))
+	if (fs_file_random_fail_begin(file, FS_OP_METADATA))
 		return -1;
-	return fs_get_metadata(file->super, metadata_r);
+	ret = fs_get_metadata(file->super, metadata_r);
+	return fs_file_random_fail_end(file, ret, FS_OP_METADATA);
 }
 
 static bool fs_randomfail_prefetch(struct fs_file *_file, uoff_t length)
 {
 	struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file;
 
-	if (fs_random_fail(_file->fs, FS_OP_PREFETCH))
+	if (fs_random_fail(_file->fs, 1, FS_OP_PREFETCH))
 		return TRUE;
 	return fs_prefetch(file->super, length);
 }
@@ -318,10 +344,12 @@ static bool fs_randomfail_prefetch(struct fs_file *_file, uoff_t length)
 static ssize_t fs_randomfail_read(struct fs_file *_file, void *buf, size_t size)
 {
 	struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file;
+	int ret;
 
-	if (fs_random_fail(_file->fs, FS_OP_READ))
+	if (fs_file_random_fail_begin(file, FS_OP_READ))
 		return -1;
-	return fs_read(file->super, buf, size);
+	ret = fs_read(file->super, buf, size);
+	return fs_file_random_fail_end(file, ret, FS_OP_READ);
 }
 
 static struct istream *
@@ -342,10 +370,12 @@ fs_randomfail_read_stream(struct fs_file *_file, size_t max_buffer_size)
 static int fs_randomfail_write(struct fs_file *_file, const void *data, size_t size)
 {
 	struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file;
+	int ret;
 
-	if (fs_random_fail(_file->fs, FS_OP_WRITE))
+	if (fs_file_random_fail_begin(file, FS_OP_WRITE))
 		return -1;
-	return fs_write(file->super, data, size);
+	ret = fs_write(file->super, data, size);
+	return fs_file_random_fail_end(file, ret, FS_OP_EXISTS);
 }
 
 static void fs_randomfail_write_stream(struct fs_file *_file)
@@ -373,10 +403,10 @@ static int fs_randomfail_write_stream_finish(struct fs_file *_file, bool success
 			_file->output = NULL;
 		else
 			o_stream_unref(&_file->output);
-	}
-	if (!success || fs_random_fail(_file->fs, FS_OP_WRITE)) {
-		fs_write_stream_abort(file->super, &file->super_output);
-		return -1;
+		if (!success || fs_random_fail(_file->fs, 1, FS_OP_WRITE)) {
+			fs_write_stream_abort(file->super, &file->super_output);
+			return -1;
+		}
 	}
 	return fs_write_stream_finish(file->super, &file->super_output);
 }
@@ -386,7 +416,7 @@ fs_randomfail_lock(struct fs_file *_file, unsigned int secs, struct fs_lock **lo
 {
 	struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file;
 
-	if (fs_random_fail(_file->fs, FS_OP_LOCK))
+	if (fs_random_fail(_file->fs, 1, FS_OP_LOCK))
 		return -1;
 	return fs_lock(file->super, secs, lock_r);
 }
@@ -399,52 +429,62 @@ static void fs_randomfail_unlock(struct fs_lock *_lock ATTR_UNUSED)
 static int fs_randomfail_exists(struct fs_file *_file)
 {
 	struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file;
+	int ret;
 
-	if (fs_random_fail(_file->fs, FS_OP_EXISTS))
+	if (fs_file_random_fail_begin(file, FS_OP_EXISTS))
 		return -1;
-	return fs_exists(file->super);
+	ret = fs_exists(file->super);
+	return fs_file_random_fail_end(file, ret, FS_OP_EXISTS);
 }
 
 static int fs_randomfail_stat(struct fs_file *_file, struct stat *st_r)
 {
 	struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file;
+	int ret;
 
-	if (fs_random_fail(_file->fs, FS_OP_STAT))
+	if (fs_file_random_fail_begin(file, FS_OP_STAT))
 		return -1;
-	return fs_stat(file->super, st_r);
+	ret = fs_stat(file->super, st_r);
+	return fs_file_random_fail_end(file, ret, FS_OP_STAT);
 }
 
 static int fs_randomfail_copy(struct fs_file *_src, struct fs_file *_dest)
 {
 	struct randomfail_fs_file *src = (struct randomfail_fs_file *)_src;
 	struct randomfail_fs_file *dest = (struct randomfail_fs_file *)_dest;
+	int ret;
 
-	if (fs_random_fail(_dest->fs, FS_OP_COPY))
+	if (fs_file_random_fail_begin(dest, FS_OP_COPY))
 		return -1;
 
 	if (_src != NULL)
-		return fs_copy(src->super, dest->super);
+		ret = fs_copy(src->super, dest->super);
 	else
-		return fs_copy_finish_async(dest->super);
+		ret = fs_copy_finish_async(dest->super);
+	return fs_file_random_fail_end(dest, ret, FS_OP_COPY);
 }
 
 static int fs_randomfail_rename(struct fs_file *_src, struct fs_file *_dest)
 {
 	struct randomfail_fs_file *src = (struct randomfail_fs_file *)_src;
 	struct randomfail_fs_file *dest = (struct randomfail_fs_file *)_dest;
+	int ret;
 
-	if (fs_random_fail(_dest->fs, FS_OP_RENAME))
+	if (fs_file_random_fail_begin(dest, FS_OP_RENAME))
 		return -1;
-	return fs_rename(src->super, dest->super);
+	ret = fs_rename(src->super, dest->super);
+	return fs_file_random_fail_end(dest, ret, FS_OP_RENAME);
 }
 
 static int fs_randomfail_delete(struct fs_file *_file)
 {
 	struct randomfail_fs_file *file = (struct randomfail_fs_file *)_file;
+	int ret;
 
-	if (fs_random_fail(_file->fs, FS_OP_DELETE))
+	if (fs_file_random_fail_begin(file, FS_OP_DELETE))
 		return -1;
-	return fs_delete(file->super);
+	ret = fs_delete(file->super);
+	return fs_file_random_fail_end(file, ret, FS_OP_DELETE);
 }
 
 static struct fs_iter *

From 6ab59f3201efb2120b66006fe00ff8d3d25e7607 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 2 May 2016 18:16:00 +0300
Subject: [PATCH 272/450] quota: Skip reading mail sizes when quota backend
 doesn't need it.

If quota backend is updating the quota internally, it's just going to ignore
the looked up size. The only reason for looking up the sizes is to check
with quota_try_alloc() whether user is going over quota.
---
 src/plugins/quota/quota-count.c   |  1 +
 src/plugins/quota/quota-dirsize.c |  9 ++++++++-
 src/plugins/quota/quota-fs.c      |  1 +
 src/plugins/quota/quota-private.h |  5 +++++
 src/plugins/quota/quota-storage.c | 20 ++++++++++++++------
 src/plugins/quota/quota.c         | 18 +++++++++++++++++-
 6 files changed, 46 insertions(+), 8 deletions(-)

diff --git a/src/plugins/quota/quota-count.c b/src/plugins/quota/quota-count.c
index 5db0525bf09..76072eaa566 100644
--- a/src/plugins/quota/quota-count.c
+++ b/src/plugins/quota/quota-count.c
@@ -206,6 +206,7 @@ static int count_quota_init(struct quota_root *root, const char *args,
 		*error_r = "quota count backend requires quota_vsizes=yes";
 		return -1;
 	}
+	root->auto_updating = TRUE;
 	return quota_root_default_init(root, args, error_r);
 }
 
diff --git a/src/plugins/quota/quota-dirsize.c b/src/plugins/quota/quota-dirsize.c
index 1d214061764..17fe6e5def3 100644
--- a/src/plugins/quota/quota-dirsize.c
+++ b/src/plugins/quota/quota-dirsize.c
@@ -25,6 +25,13 @@ static struct quota_root *dirsize_quota_alloc(void)
 	return i_new(struct quota_root, 1);
 }
 
+static int dirsize_quota_init(struct quota_root *root, const char *args,
+			      const char **error_r)
+{
+	root->auto_updating = TRUE;
+	return quota_root_default_init(root, args, error_r);
+}
+
 static void dirsize_quota_deinit(struct quota_root *_root)
 {
 	i_free(_root);
@@ -210,7 +217,7 @@ struct quota_backend quota_backend_dirsize = {
 
 	{
 		dirsize_quota_alloc,
-		NULL,
+		dirsize_quota_init,
 		dirsize_quota_deinit,
 		NULL,
 		NULL,
diff --git a/src/plugins/quota/quota-fs.c b/src/plugins/quota/quota-fs.c
index 5b3431a5a86..b3c15be3a88 100644
--- a/src/plugins/quota/quota-fs.c
+++ b/src/plugins/quota/quota-fs.c
@@ -122,6 +122,7 @@ static int fs_quota_init(struct quota_root *_root, const char *args,
 			return -1;
 		}
 	}
+	_root->auto_updating = TRUE;
 	return 0;
 }
 
diff --git a/src/plugins/quota/quota-private.h b/src/plugins/quota/quota-private.h
index 60683f7d89d..3341c646a75 100644
--- a/src/plugins/quota/quota-private.h
+++ b/src/plugins/quota/quota-private.h
@@ -137,6 +137,9 @@ struct quota_root {
 
 	/* don't enforce quota when saving */
 	unsigned int no_enforcing:1;
+	/* quota is automatically updated. update() should be called but the
+	   bytes/count won't be used. */
+	unsigned int auto_updating:1;
 	/* If user has unlimited quota, disable quota tracking */
 	unsigned int disable_unlimited_tracking:1;
 	/* Set while quota is being recalculated to avoid recursion. */
@@ -177,6 +180,8 @@ struct quota_transaction_context {
 	unsigned int failed:1;
 	unsigned int recalculate:1;
 	unsigned int sync_transaction:1;
+	/* TRUE if all roots have auto_updating=TRUE */
+	unsigned int auto_updating:1;
 };
 
 /* Register storage to all user's quota roots. */
diff --git a/src/plugins/quota/quota-storage.c b/src/plugins/quota/quota-storage.c
index 4d75a319437..1b47bb8f717 100644
--- a/src/plugins/quota/quota-storage.c
+++ b/src/plugins/quota/quota-storage.c
@@ -52,9 +52,15 @@ static void quota_mail_expunge(struct mail *_mail)
 	struct quota_mailbox *qbox = QUOTA_CONTEXT(_mail->box);
 	struct quota_user *quser = QUOTA_USER_CONTEXT(_mail->box->storage->user);
 	union mail_module_context *qmail = QUOTA_MAIL_CONTEXT(mail);
+	struct quota_transaction_context *qt = QUOTA_CONTEXT(_mail->transaction);
 	uoff_t size;
 	int ret;
 
+	if (qt->auto_updating) {
+		qmail->super.expunge(_mail);
+		return;
+	}
+
 	/* We need to handle the situation where multiple transactions expunged
 	   the mail at the same time. In here we'll just save the message's
 	   physical size and do the quota freeing later when the message was
@@ -338,6 +344,14 @@ static void quota_mailbox_sync_notify(struct mailbox *box, uint32_t uid,
 		return;
 	}
 
+	if (qbox->expunge_qt == NULL) {
+		qbox->expunge_qt = quota_transaction_begin(box);
+		qbox->expunge_qt->sync_transaction =
+			qbox->sync_transaction_expunge;
+	}
+	if (qbox->expunge_qt->auto_updating)
+		return;
+
 	/* we're in the middle of syncing the mailbox, so it's a bad idea to
 	   try and get the message sizes at this point. Rely on sizes that
 	   we saved earlier, or recalculate the whole quota if we don't know
@@ -362,12 +376,6 @@ static void quota_mailbox_sync_notify(struct mailbox *box, uint32_t uid,
 		qbox->prev_idx = i;
 	}
 
-	if (qbox->expunge_qt == NULL) {
-		qbox->expunge_qt = quota_transaction_begin(box);
-		qbox->expunge_qt->sync_transaction =
-			qbox->sync_transaction_expunge;
-	}
-
 	if (i != count) {
 		/* we already know the size */
 		sizep = array_idx(&qbox->expunge_sizes, i);
diff --git a/src/plugins/quota/quota.c b/src/plugins/quota/quota.c
index 4e356f0216b..f4ce4495969 100644
--- a/src/plugins/quota/quota.c
+++ b/src/plugins/quota/quota.c
@@ -752,6 +752,7 @@ int quota_set_resource(struct quota_root *root, const char *name,
 struct quota_transaction_context *quota_transaction_begin(struct mailbox *box)
 {
 	struct quota_transaction_context *ctx;
+	struct quota_root *const *rootp;
 
 	ctx = i_new(struct quota_transaction_context, 1);
 	ctx->quota = box->list->ns->owner != NULL ?
@@ -764,6 +765,12 @@ struct quota_transaction_context *quota_transaction_begin(struct mailbox *box)
 	ctx->bytes_ceil2 = (uint64_t)-1;
 	ctx->count_ceil = (uint64_t)-1;
 
+	ctx->auto_updating = TRUE;
+	array_foreach(&ctx->quota->roots, rootp) {
+		if (!(*rootp)->auto_updating)
+			ctx->auto_updating = FALSE;
+	}
+
 	if (box->storage->user->dsyncing) {
 		/* ignore quota for dsync */
 		ctx->limits_set = TRUE;
@@ -1092,7 +1099,12 @@ int quota_try_alloc(struct quota_transaction_context *ctx,
 	ret = quota_test_alloc(ctx, size, too_large_r);
 	if (ret <= 0)
 		return ret;
-
+	/* with quota_try_alloc() we want to keep track of how many bytes
+	   we've been adding/removing, so disable auto_updating=TRUE
+	   optimization. this of course doesn't work perfectly if
+	   quota_alloc() or quota_free*() was already used within the same
+	   transaction, but that doesn't normally happen. */
+	ctx->auto_updating = FALSE;
 	quota_alloc(ctx, mail);
 	return 1;
 }
@@ -1152,6 +1164,8 @@ void quota_alloc(struct quota_transaction_context *ctx, struct mail *mail)
 {
 	uoff_t size;
 
+	if (ctx->auto_updating)
+		return;
 	if (mail_get_physical_size(mail, &size) == 0)
 		ctx->bytes_used += size;
 
@@ -1163,6 +1177,8 @@ void quota_free(struct quota_transaction_context *ctx, struct mail *mail)
 {
 	uoff_t size;
 
+	if (ctx->auto_updating)
+		return;
 	if (mail_get_physical_size(mail, &size) < 0)
 		quota_recalculate(ctx);
 	else

From 4441f95b3ea595fae9a10196c985d21ffe7dc8ec Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 2 May 2016 18:27:02 +0300
Subject: [PATCH 273/450] quota: Differentiate between forced and non-forced
 quota recalc

The "count" backend doesn't need to recalc quota unless an explicit "doveadm
quota recalc" command is called.
---
 src/plugins/quota/doveadm-quota.c |  2 +-
 src/plugins/quota/quota-count.c   |  2 +-
 src/plugins/quota/quota-dict.c    |  2 +-
 src/plugins/quota/quota-maildir.c |  2 +-
 src/plugins/quota/quota-private.h |  2 +-
 src/plugins/quota/quota-storage.c |  2 +-
 src/plugins/quota/quota.c         |  9 +++++----
 src/plugins/quota/quota.h         | 14 +++++++++++++-
 8 files changed, 24 insertions(+), 11 deletions(-)

diff --git a/src/plugins/quota/doveadm-quota.c b/src/plugins/quota/doveadm-quota.c
index cbf96d4bb8c..2bdf7dc1465 100644
--- a/src/plugins/quota/doveadm-quota.c
+++ b/src/plugins/quota/doveadm-quota.c
@@ -101,7 +101,7 @@ cmd_quota_recalc_run(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED,
 
 	memset(&trans, 0, sizeof(trans));
 	trans.quota = quser->quota;
-	trans.recalculate = TRUE;
+	trans.recalculate = QUOTA_RECALCULATE_FORCED;
 
 	array_foreach(&quser->quota->roots, root)
 		(void)(*root)->backend.v.update(*root, &trans);
diff --git a/src/plugins/quota/quota-count.c b/src/plugins/quota/quota-count.c
index 76072eaa566..246c46dc589 100644
--- a/src/plugins/quota/quota-count.c
+++ b/src/plugins/quota/quota-count.c
@@ -310,7 +310,7 @@ count_quota_update(struct quota_root *root,
 	struct count_quota_root *croot = (struct count_quota_root *)root;
 
 	croot->cache_timeval.tv_sec = 0;
-	if (ctx->recalculate) {
+	if (ctx->recalculate == QUOTA_RECALCULATE_FORCED) {
 		if (quota_count_recalculate(root) < 0)
 			return -1;
 	}
diff --git a/src/plugins/quota/quota-dict.c b/src/plugins/quota/quota-dict.c
index 4026faa149f..1394a228aa1 100644
--- a/src/plugins/quota/quota-dict.c
+++ b/src/plugins/quota/quota-dict.c
@@ -217,7 +217,7 @@ dict_quota_update(struct quota_root *_root,
 	struct dict_transaction_context *dt;
 	uint64_t value;
 
-	if (ctx->recalculate) {
+	if (ctx->recalculate != QUOTA_RECALCULATE_DONT) {
 		if (dict_quota_count(root, TRUE, &value) < 0)
 			return -1;
 	} else {
diff --git a/src/plugins/quota/quota-maildir.c b/src/plugins/quota/quota-maildir.c
index df745fc0c10..7c7a23f7a58 100644
--- a/src/plugins/quota/quota-maildir.c
+++ b/src/plugins/quota/quota-maildir.c
@@ -903,7 +903,7 @@ maildir_quota_update(struct quota_root *_root,
 		   we wanted to do. */
 	} else if (root->fd == -1)
 		(void)maildirsize_recalculate(root);
-	else if (ctx->recalculate) {
+	else if (ctx->recalculate != QUOTA_RECALCULATE_DONT) {
 		i_close_fd(&root->fd);
 		(void)maildirsize_recalculate(root);
 	} else if (maildirsize_update(root, ctx->count_used, ctx->bytes_used) < 0) {
diff --git a/src/plugins/quota/quota-private.h b/src/plugins/quota/quota-private.h
index 3341c646a75..6ac87af181b 100644
--- a/src/plugins/quota/quota-private.h
+++ b/src/plugins/quota/quota-private.h
@@ -175,10 +175,10 @@ struct quota_transaction_context {
 	uint64_t bytes_over, count_over;
 
 	struct mail *tmp_mail;
+	enum quota_recalculate recalculate;
 
 	unsigned int limits_set:1;
 	unsigned int failed:1;
-	unsigned int recalculate:1;
 	unsigned int sync_transaction:1;
 	/* TRUE if all roots have auto_updating=TRUE */
 	unsigned int auto_updating:1;
diff --git a/src/plugins/quota/quota-storage.c b/src/plugins/quota/quota-storage.c
index 1b47bb8f717..14270d8df28 100644
--- a/src/plugins/quota/quota-storage.c
+++ b/src/plugins/quota/quota-storage.c
@@ -416,7 +416,7 @@ static void quota_mailbox_sync_notify(struct mailbox *box, uint32_t uid,
 			index_mailbox_vsize_hdr_expunge(ibox->vsize_update, uid, size);
 	} else {
 		/* there's no way to get the size. recalculate the quota. */
-		quota_recalculate(qbox->expunge_qt);
+		quota_recalculate(qbox->expunge_qt, QUOTA_RECALCULATE_MISSING_FREES);
 		qbox->recalculate = TRUE;
 	}
 }
diff --git a/src/plugins/quota/quota.c b/src/plugins/quota/quota.c
index f4ce4495969..6d881e1fd33 100644
--- a/src/plugins/quota/quota.c
+++ b/src/plugins/quota/quota.c
@@ -955,7 +955,7 @@ int quota_transaction_commit(struct quota_transaction_context **_ctx)
 	if (ctx->failed)
 		ret = -1;
 	else if (ctx->bytes_used != 0 || ctx->count_used != 0 ||
-		 ctx->recalculate) T_BEGIN {
+		 ctx->recalculate != QUOTA_RECALCULATE_DONT) T_BEGIN {
 		ARRAY(struct quota_root *) warn_roots;
 
 		mailbox_name = mailbox_get_vname(ctx->box);
@@ -1180,7 +1180,7 @@ void quota_free(struct quota_transaction_context *ctx, struct mail *mail)
 	if (ctx->auto_updating)
 		return;
 	if (mail_get_physical_size(mail, &size) < 0)
-		quota_recalculate(ctx);
+		quota_recalculate(ctx, QUOTA_RECALCULATE_MISSING_FREES);
 	else
 		quota_free_bytes(ctx, size);
 }
@@ -1192,7 +1192,8 @@ void quota_free_bytes(struct quota_transaction_context *ctx,
 	ctx->count_used--;
 }
 
-void quota_recalculate(struct quota_transaction_context *ctx)
+void quota_recalculate(struct quota_transaction_context *ctx,
+		       enum quota_recalculate recalculate)
 {
-	ctx->recalculate = TRUE;
+	ctx->recalculate = recalculate;
 }
diff --git a/src/plugins/quota/quota.h b/src/plugins/quota/quota.h
index 04847d1f90d..1088f5a9b05 100644
--- a/src/plugins/quota/quota.h
+++ b/src/plugins/quota/quota.h
@@ -19,6 +19,17 @@ struct quota_root;
 struct quota_root_iter;
 struct quota_transaction_context;
 
+enum quota_recalculate {
+	QUOTA_RECALCULATE_DONT = 0,
+	/* We may want to recalculate quota because we weren't able to call
+	   quota_free*() correctly for all mails. Quota needs to be
+	   recalculated unless the backend does the quota tracking
+	   internally. */
+	QUOTA_RECALCULATE_MISSING_FREES,
+	/* doveadm quota recalc called - make sure the quota is correct */
+	QUOTA_RECALCULATE_FORCED
+};
+
 int quota_user_read_settings(struct mail_user *user,
 			     struct quota_settings **set_r,
 			     const char **error_r);
@@ -81,7 +92,8 @@ void quota_free(struct quota_transaction_context *ctx, struct mail *mail);
 void quota_free_bytes(struct quota_transaction_context *ctx,
 		      uoff_t physical_size);
 /* Mark the quota to be recalculated */
-void quota_recalculate(struct quota_transaction_context *ctx);
+void quota_recalculate(struct quota_transaction_context *ctx,
+		       enum quota_recalculate recalculate);
 
 /* Execute quota_over_scripts if needed. */
 void quota_over_flag_check_startup(struct quota *quota);

From 318974e640643fed09abfbe450df727508078db4 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 5 Jun 2016 16:01:05 +0300
Subject: [PATCH 274/450] mbox: Code cleanup - use bool instead of int for
 tracking locked-status

---
 src/lib-storage/index/mbox/mbox-lock.c | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/lib-storage/index/mbox/mbox-lock.c b/src/lib-storage/index/mbox/mbox-lock.c
index 647f33611f6..9b6ff4ffc45 100644
--- a/src/lib-storage/index/mbox/mbox-lock.c
+++ b/src/lib-storage/index/mbox/mbox-lock.c
@@ -41,7 +41,7 @@ enum mbox_dotlock_op {
 
 struct mbox_lock_context {
 	struct mbox_mailbox *mbox;
-	int lock_status[MBOX_LOCK_COUNT];
+	bool locked_status[MBOX_LOCK_COUNT];
 	bool checked_file;
 
 	int lock_type;
@@ -674,7 +674,8 @@ mbox_lock_list(struct mbox_lock_context *ctx, int lock_type,
 {
 	enum mbox_lock_type *lock_types;
         enum mbox_lock_type type;
-	int i, ret = 0, lock_status;
+	int i, ret = 0;
+	bool locked_status;
 
 	ctx->lock_type = lock_type;
 
@@ -684,11 +685,11 @@ mbox_lock_list(struct mbox_lock_context *ctx, int lock_type,
 		ctx->mbox->storage->read_locks;
 	for (i = idx; lock_types[i] != (enum mbox_lock_type)-1; i++) {
 		type = lock_types[i];
-		lock_status = lock_type != F_UNLCK;
+		locked_status = lock_type != F_UNLCK;
 
-		if (ctx->lock_status[type] == lock_status)
+		if (ctx->locked_status[type] == locked_status)
 			continue;
-		ctx->lock_status[type] = lock_status;
+		ctx->locked_status[type] = locked_status;
 
 		ret = lock_data[type].func(ctx, lock_type, max_wait_time);
 		if (ret <= 0)
@@ -733,9 +734,9 @@ static int mbox_update_locking(struct mbox_mailbox *mbox, int lock_type,
                         mbox->storage->read_locks;
 
 		for (i = 0; i < MBOX_LOCK_COUNT; i++)
-			ctx.lock_status[i] = 1;
+			ctx.locked_status[i] = TRUE;
 		for (i = 0; read_locks[i] != (enum mbox_lock_type)-1; i++)
-			ctx.lock_status[read_locks[i]] = 0;
+			ctx.locked_status[read_locks[i]] = FALSE;
 		drop_locks = TRUE;
 	} else {
 		drop_locks = FALSE;
@@ -761,11 +762,11 @@ static int mbox_update_locking(struct mbox_mailbox *mbox, int lock_type,
 		const enum mbox_lock_type *write_locks =
 			mbox->storage->write_locks;
 
-		memset(ctx.lock_status, 0, sizeof(ctx.lock_status));
+		memset(ctx.locked_status, 0, sizeof(ctx.locked_status));
 		for (i = 0; write_locks[i] != (enum mbox_lock_type)-1; i++)
-			ctx.lock_status[write_locks[i]] = 1;
+			ctx.locked_status[write_locks[i]] = TRUE;
 		for (i = 0; read_locks[i] != (enum mbox_lock_type)-1; i++)
-			ctx.lock_status[read_locks[i]] = 0;
+			ctx.locked_status[read_locks[i]] = FALSE;
 
 		mbox->mbox_lock_type = F_WRLCK;
 		mbox_lock_list(&ctx, F_UNLCK, 0, 0);
@@ -877,7 +878,7 @@ int mbox_unlock(struct mbox_mailbox *mbox, unsigned int lock_id)
 	ctx.mbox = mbox;
 
 	for (i = 0; i < MBOX_LOCK_COUNT; i++)
-		ctx.lock_status[i] = 1;
+		ctx.locked_status[i] = TRUE;
 
 	return mbox_unlock_files(&ctx);
 }

From 4a5eb0c4c0c0a1a1491a27e0cb5d88f96c1cb0e8 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 7 Jun 2016 03:04:32 +0300
Subject: [PATCH 275/450] lib-lda: Don't assert-crash if home directory isn't
 set.

---
 src/lib-lda/duplicate.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/lib-lda/duplicate.c b/src/lib-lda/duplicate.c
index 65040c408e4..609cf32fbfb 100644
--- a/src/lib-lda/duplicate.c
+++ b/src/lib-lda/duplicate.c
@@ -343,7 +343,8 @@ struct duplicate_context *duplicate_init(struct mail_user *user)
 	}
 
 	ctx = i_new(struct duplicate_context, 1);
-	ctx->path = i_strconcat(home, "/"DUPLICATE_FNAME, NULL);
+	ctx->path = home == NULL ? NULL :
+		i_strconcat(home, "/"DUPLICATE_FNAME, NULL);
 	ctx->dotlock_set = default_duplicate_dotlock_set;
 
 	mail_set = mail_user_set_get_storage_set(user);

From 9a7dc9132bcd335d64d7a78c18416d91fe4293cb Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 7 Jun 2016 03:25:16 +0300
Subject: [PATCH 276/450] lib-dcrypt: Fixed library dependencies

_DEPENDENCIES needs an explicit paths, not "-lssl" and such. Fixes building
with e.g. OSX.
---
 src/lib-dcrypt/Makefile.am | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/lib-dcrypt/Makefile.am b/src/lib-dcrypt/Makefile.am
index 42fe80848a5..9480f64f7df 100644
--- a/src/lib-dcrypt/Makefile.am
+++ b/src/lib-dcrypt/Makefile.am
@@ -16,7 +16,6 @@ libdcrypt_la_CFLAGS = $(AM_CPPFLAGS) \
 if BUILD_OPENSSL
 pkglib_LTLIBRARIES += libdcrypt_openssl.la
 libdcrypt_openssl_la_SOURCES = dcrypt-openssl.c
-libdcrypt_openssl_la_DEPENDENCIES = $(SSL_LIBS)
 libdcrypt_openssl_la_LDFLAGS = -module -avoid-version -shared $(SSL_LIBS)
 libdcrypt_openssl_la_CFLAGS = $(AM_CPPFLAGS) \
 	$(SSL_CFLAGS)

From 8fdb634ad750b58b2b4a719124cc96c5321e625e Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Tue, 7 Jun 2016 08:50:12 +0300
Subject: [PATCH 277/450] doveadm-server: Disable idle timeout for now

---
 src/doveadm/client-connection-http.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/doveadm/client-connection-http.c b/src/doveadm/client-connection-http.c
index 30c6d19610e..52191ebcb12 100644
--- a/src/doveadm/client-connection-http.c
+++ b/src/doveadm/client-connection-http.c
@@ -757,7 +757,7 @@ static void doveadm_http_server_send_response(void *context)
 }
 
 static const struct http_server_settings http_server_set = {
-	.max_client_idle_time_msecs = 5000,
+	.max_client_idle_time_msecs = 0,
         .max_pipelined_requests = 0
 };
 

From 5aa66452ffd9797e9da31b2741ad68d4a9dfab15 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sat, 11 Jun 2016 17:13:12 +0300
Subject: [PATCH 278/450] cassandra: Try fallback_consistency also for write
 timeout failures.

---
 src/lib-sql/driver-cassandra.c | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/lib-sql/driver-cassandra.c b/src/lib-sql/driver-cassandra.c
index 05cd3d97592..78595ceb834 100644
--- a/src/lib-sql/driver-cassandra.c
+++ b/src/lib-sql/driver-cassandra.c
@@ -707,7 +707,16 @@ static void query_callback(CassFuture *future, void *context)
 		result->error = i_strdup_printf("Query '%s' failed: %.*s",
 						result->query,
 						(int)errsize, errmsg);
-		if (error == CASS_ERROR_SERVER_UNAVAILABLE &&
+		/* unavailable = cassandra server knows that there aren't
+		   enough nodes available.
+
+		   write timeout = cassandra server couldn't reach all the
+		   needed nodes. this may be because it hasn't yet detected
+		   that the servers are down, or because the servers are just
+		   too busy. we'll try the fallback consistency to avoid
+		   unnecessary temporary errors. */
+		if ((error == CASS_ERROR_SERVER_UNAVAILABLE ||
+		     error == CASS_ERROR_SERVER_WRITE_TIMEOUT) &&
 		    result->fallback_consistency != result->consistency) {
 			/* retry with fallback consistency */
 			query_resend_with_fallback(result);

From 62fc7d0cce0ab7f869ba79637dd7b6e04df71824 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sat, 11 Jun 2016 20:50:39 +0300
Subject: [PATCH 279/450] lib-storage: mail_set_cache_corrupted*() now sets
 internal error to storage.

At least index_mail_set_message_parts_corrupted() assumed that this was
being done.
---
 src/lib-storage/index/index-mail.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lib-storage/index/index-mail.c b/src/lib-storage/index/index-mail.c
index 856822e71f3..1c37e74afa3 100644
--- a/src/lib-storage/index/index-mail.c
+++ b/src/lib-storage/index/index-mail.c
@@ -2197,6 +2197,7 @@ void index_mail_set_cache_corrupted_reason(struct mail *mail,
 			"Broken %s for mail UID %u in mailbox %s: %s",
 			field_name, mail->uid, mail->box->vname, reason);
 	}
+	mail_storage_set_internal_error(mail->box->storage);
 }
 
 int index_mail_opened(struct mail *mail ATTR_UNUSED,

From 2a8dbe2b5e5c8340349f50bd008d84b8a3bd6f37 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sat, 11 Jun 2016 21:20:37 +0300
Subject: [PATCH 280/450] lib-mail: message-header-parser now keeps istream
 referenced.

---
 src/lib-mail/message-header-parser.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/lib-mail/message-header-parser.c b/src/lib-mail/message-header-parser.c
index 449667d750f..587048fc589 100644
--- a/src/lib-mail/message-header-parser.c
+++ b/src/lib-mail/message-header-parser.c
@@ -33,6 +33,7 @@ message_parse_header_init(struct istream *input, struct message_size *hdr_size,
 	ctx->name = str_new(default_pool, 128);
 	ctx->flags = flags;
 	ctx->value_buf = buffer_create_dynamic(default_pool, 4096);
+	i_stream_ref(input);
 
 	if (hdr_size != NULL)
 		memset(hdr_size, 0, sizeof(*hdr_size));
@@ -43,6 +44,7 @@ void message_parse_header_deinit(struct message_header_parser_ctx **_ctx)
 {
 	struct message_header_parser_ctx *ctx = *_ctx;
 
+	i_stream_unref(&ctx->input);
 	buffer_free(&ctx->value_buf);
 	str_free(&ctx->name);
 	i_free(ctx);

From 90c8dbd884db01107f87ee9a0d23fc39e7fc195f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sat, 11 Jun 2016 21:23:07 +0300
Subject: [PATCH 281/450] lib-mail: Fixed message_parser_init_from_parts() with
 truncated MIME headers

---
 src/lib-mail/message-parser.c | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/lib-mail/message-parser.c b/src/lib-mail/message-parser.c
index b31b494e757..20ad1027256 100644
--- a/src/lib-mail/message-parser.c
+++ b/src/lib-mail/message-parser.c
@@ -1005,14 +1005,22 @@ static int preparsed_parse_next_header(struct message_parser_ctx *ctx,
 static int preparsed_parse_next_header_init(struct message_parser_ctx *ctx,
 					    struct message_block *block_r)
 {
+	struct istream *hdr_input;
+
 	i_assert(ctx->hdr_parser_ctx == NULL);
 
 	i_assert(ctx->part->physical_pos >= ctx->input->v_offset);
 	i_stream_skip(ctx->input, ctx->part->physical_pos -
 		      ctx->input->v_offset);
 
+	/* the header may become truncated by --boundaries. limit the header
+	   stream's size to what it's supposed to be to avoid duplicating (and
+	   keeping in sync!) all the same complicated logic as in
+	   parse_next_header(). */
+	hdr_input = i_stream_create_limit(ctx->input, ctx->part->header_size.physical_size);
 	ctx->hdr_parser_ctx =
-		message_parse_header_init(ctx->input, NULL, ctx->hdr_flags);
+		message_parse_header_init(hdr_input, NULL, ctx->hdr_flags);
+	i_stream_unref(&hdr_input);
 
 	ctx->parse_next_block = preparsed_parse_next_header;
 	return preparsed_parse_next_header(ctx, block_r);

From 1f2c248dc374563a50beaab5024383541377612d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sat, 11 Jun 2016 21:23:57 +0300
Subject: [PATCH 282/450] lib-mail: Improved message-parser unit tests.

---
 src/lib-mail/test-message-parser.c | 44 +++++++++++++++++++++++++++++-
 1 file changed, 43 insertions(+), 1 deletion(-)

diff --git a/src/lib-mail/test-message-parser.c b/src/lib-mail/test-message-parser.c
index 29fce70726a..66360d49053 100644
--- a/src/lib-mail/test-message-parser.c
+++ b/src/lib-mail/test-message-parser.c
@@ -68,6 +68,32 @@ static bool msg_parts_cmp(struct message_part *p1, struct message_part *p2)
 	return TRUE;
 }
 
+static void test_parsed_parts(struct istream *input, struct message_part *parts)
+{
+	struct message_parser_ctx *parser;
+	struct message_block block;
+	struct message_part *parts2;
+	uoff_t i, input_size;
+	const char *error;
+	int ret;
+
+	i_stream_seek(input, 0);
+	if (i_stream_get_size(input, TRUE, &input_size) < 0)
+		i_unreached();
+
+	parser = message_parser_init_from_parts(parts, input, 0,
+					MESSAGE_PARSER_FLAG_SKIP_BODY_BLOCK);
+	for (i = 1; i <= input_size*2+1; i++) {
+		test_istream_set_size(input, i/2);
+		if (i > TEST_MSG_LEN*2)
+			test_istream_set_allow_eof(input, TRUE);
+		while ((ret = message_parser_parse_next_block(parser,
+							      &block)) > 0) ;
+	}
+	test_assert(message_parser_deinit_from_parts(&parser, &parts2, &error) == 0);
+	test_assert(msg_parts_cmp(parts, parts2));
+}
+
 static void test_message_parser_small_blocks(void)
 {
 	struct message_parser_ctx *parser;
@@ -170,18 +196,25 @@ static const char input_msg[] =
 	test_assert(message_parser_deinit(&parser, &parts) == 0);
 
 	test_assert((parts->flags & MESSAGE_PART_FLAG_MULTIPART) != 0);
+	test_assert(parts->header_size.lines == 2);
+	test_assert(parts->header_size.physical_size == 48);
+	test_assert(parts->header_size.virtual_size == 48+2);
 	test_assert(parts->body_size.lines == 8);
 	test_assert(parts->body_size.physical_size == 112);
 	test_assert(parts->body_size.virtual_size == 112+7);
+	test_assert(parts->children->physical_pos == 55);
 	test_assert(parts->children->header_size.physical_size == 0);
 	test_assert(parts->children->body_size.physical_size == 0);
 	test_assert(parts->children->body_size.lines == 0);
+	test_assert(parts->children->next->physical_pos == 62);
 	test_assert(parts->children->next->header_size.physical_size == 24);
 	test_assert(parts->children->next->header_size.virtual_size == 24);
 	test_assert(parts->children->next->header_size.lines == 0);
+	test_assert(parts->children->next->next->physical_pos == 94);
 	test_assert(parts->children->next->next->header_size.physical_size == 24);
 	test_assert(parts->children->next->next->header_size.virtual_size == 24);
 	test_assert(parts->children->next->next->header_size.lines == 0);
+	test_assert(parts->children->next->next->next->physical_pos == 127);
 	test_assert(parts->children->next->next->next->header_size.physical_size == 23);
 	test_assert(parts->children->next->next->next->header_size.virtual_size == 23);
 	test_assert(parts->children->next->next->next->header_size.lines == 0);
@@ -191,6 +224,7 @@ static const char input_msg[] =
 	}
 	test_assert(parts->children->next->next->next->next == NULL);
 
+	test_parsed_parts(input, parts);
 	i_stream_unref(&input);
 	pool_unref(&pool);
 	test_end();
@@ -251,6 +285,7 @@ static const char input_msg[] =
 	test_assert(parts->children->next->body_size.virtual_size == 5+2);
 	test_assert(parts->children->next->children == NULL);
 
+	test_parsed_parts(input, parts);
 	i_stream_unref(&input);
 	pool_unref(&pool);
 	test_end();
@@ -285,6 +320,7 @@ static const char input_msg[] =
 
 	test_assert(parts->children == NULL);
 
+	test_parsed_parts(input, parts);
 	i_stream_unref(&input);
 	pool_unref(&pool);
 	test_end();
@@ -303,7 +339,7 @@ static const char input_msg[] =
 	pool_t pool;
 	int ret;
 
-	test_begin("message parser truncated mime headers 3");
+	test_begin("message parser empty multipart");
 	pool = pool_alloconly_create("message parser", 10240);
 	input = test_istream_create(input_msg);
 
@@ -322,6 +358,7 @@ static const char input_msg[] =
 
 	test_assert(parts->children == NULL);
 
+	test_parsed_parts(input, parts);
 	i_stream_unref(&input);
 	pool_unref(&pool);
 	test_end();
@@ -379,6 +416,7 @@ static const char input_msg[] =
 	test_assert(parts->children->children->body_size.physical_size == 5);
 	test_assert(parts->children->children->body_size.virtual_size == 5+1);
 
+	test_parsed_parts(input, parts);
 	i_stream_unref(&input);
 	pool_unref(&pool);
 	test_end();
@@ -436,6 +474,7 @@ static const char input_msg[] =
 	test_assert(parts->children->children->body_size.physical_size == 5);
 	test_assert(parts->children->children->body_size.virtual_size == 5+1);
 
+	test_parsed_parts(input, parts);
 	i_stream_unref(&input);
 	pool_unref(&pool);
 	test_end();
@@ -493,6 +532,7 @@ static const char input_msg[] =
 	test_assert(parts->children->children->body_size.physical_size == 5);
 	test_assert(parts->children->children->body_size.virtual_size == 5+1);
 
+	test_parsed_parts(input, parts);
 	i_stream_unref(&input);
 	pool_unref(&pool);
 	test_end();
@@ -568,6 +608,7 @@ static const char input_msg[] =
 	test_assert(part->children == NULL);
 	test_assert(part->next == NULL);
 
+	test_parsed_parts(input, parts);
 	i_stream_unref(&input);
 	pool_unref(&pool);
 	test_end();
@@ -595,6 +636,7 @@ static void test_message_parser_no_eoh(void)
 	test_assert(message_parser_parse_next_block(parser, &block) < 0);
 	test_assert(message_parser_deinit(&parser, &parts) == 0);
 
+	test_parsed_parts(input, parts);
 	i_stream_unref(&input);
 	pool_unref(&pool);
 	test_end();

From fb252d6ca28781af2a6396860ef4c1d89ea25ee7 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Sun, 12 Jun 2016 18:57:10 +0300
Subject: [PATCH 283/450] lib-dcrypt: Fix various problems

---
 src/lib-dcrypt/Makefile.am      |   5 +-
 src/lib-dcrypt/dcrypt-openssl.c | 217 +++++++++++++++++++-------------
 src/lib-dcrypt/dcrypt-private.h |   2 +
 src/lib-dcrypt/dcrypt.c         |   8 ++
 src/lib-dcrypt/dcrypt.h         |   2 +
 5 files changed, 145 insertions(+), 89 deletions(-)

diff --git a/src/lib-dcrypt/Makefile.am b/src/lib-dcrypt/Makefile.am
index 9480f64f7df..1ea6f74d21d 100644
--- a/src/lib-dcrypt/Makefile.am
+++ b/src/lib-dcrypt/Makefile.am
@@ -35,12 +35,13 @@ EXTRA_DIST = \
 	sample-v1.asc \
 	sample-v2.asc
 
-noinst_PROGRAMS = test-crypto test-stream
+test_programs = test-crypto test-stream
+noinst_PROGRAMS = $(test_programs)
 
 check: check-am check-test
 
 check-test: all-am
-	for bin in $(check_PROGRAMS); do \
+	for bin in $(test_programs); do \
 	  if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
 	done
 
diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index 797214026f2..743c345c080 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -112,6 +112,10 @@ bool dcrypt_openssl_public_key_id(struct dcrypt_public_key *key, const char *alg
 static
 bool dcrypt_openssl_public_key_id_old(struct dcrypt_public_key *key, buffer_t *result, const char **error_r);
 static
+bool dcrypt_openssl_private_key_id(struct dcrypt_private_key *key, const char *algorithm, buffer_t *result, const char **error_r);
+static
+bool dcrypt_openssl_private_key_id_old(struct dcrypt_private_key *key, buffer_t *result, const char **error_r);
+static
 bool dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r, const char **error_r ATTR_UNUSED);
 static
 void dcrypt_openssl_free_private_key(struct dcrypt_private_key **key);
@@ -675,21 +679,19 @@ bool dcrypt_openssl_pbkdf2(const unsigned char *password, size_t password_len, c
 	i_assert(rounds > 0);
 	i_assert(result_len > 0);
 	i_assert(result != NULL);
-	T_BEGIN {
-		/* determine MD */
-		const EVP_MD* md = EVP_get_digestbyname(hash);
-		if (md == NULL) {
-			if (error_r != NULL)
-				*error_r = t_strdup_printf("Invalid digest %s", hash);
-			return FALSE;
-		}
+	/* determine MD */
+	const EVP_MD* md = EVP_get_digestbyname(hash);
+	if (md == NULL) {
+		if (error_r != NULL)
+			*error_r = t_strdup_printf("Invalid digest %s", hash);
+		return FALSE;
+	}
 
-		unsigned char buffer[result_len];
-		if ((ret = PKCS5_PBKDF2_HMAC((const char*)password, password_len, salt, salt_len, rounds,
-						md, result_len, buffer)) == 1) {
-			buffer_append(result, buffer, result_len);
-		}
-	} T_END;
+	unsigned char buffer[result_len];
+	if ((ret = PKCS5_PBKDF2_HMAC((const char*)password, password_len, salt, salt_len, rounds,
+					md, result_len, buffer)) == 1) {
+		buffer_append(result, buffer, result_len);
+	}
 	if (ret != 1) return dcrypt_openssl_error(error_r);
 	return TRUE;
 }
@@ -1127,10 +1129,7 @@ bool dcrypt_openssl_load_private_key_dovecot_v2(struct dcrypt_private_key **key_
 	}
 
 	/* finally compare key to key id */
-	struct dcrypt_public_key *pubkey = NULL;
-	dcrypt_openssl_private_to_public_key(*key_r, &pubkey, NULL);
-	dcrypt_openssl_public_key_id(pubkey, "sha256", key_data, NULL);
-	dcrypt_openssl_free_public_key(&pubkey);
+	dcrypt_openssl_private_key_id(*key_r, "sha256", key_data, NULL);
 
 	if (strcmp(binary_to_hex(key_data->data, key_data->used), input[len-1]) != 0) {
 		dcrypt_openssl_free_private_key(key_r);
@@ -1149,24 +1148,22 @@ bool dcrypt_openssl_load_private_key_dovecot(struct dcrypt_private_key **key_r,
 	const char **error_r)
 {
 	bool ret;
-	T_BEGIN {
-		const char **input = t_strsplit_tab(data);
-		size_t len;
-		for(len=0;input[len]!=NULL;len++);
-		if (len < 4) {
-			if (error_r != NULL)
-				*error_r = "Corrupted data";
-			ret = FALSE;
-		} else if (*(input[0])== '1')
-			ret = dcrypt_openssl_load_private_key_dovecot_v1(key_r, len, input, password, key, error_r);
-		else if (*(input[0])== '2')
-			ret = dcrypt_openssl_load_private_key_dovecot_v2(key_r, len, input, password, key, error_r);
-		else {
-			if (error_r != NULL)
-				*error_r = "Unsupported key version";
-			ret = FALSE;
-		}
-	} T_END;
+	const char **input = t_strsplit_tab(data);
+	size_t len = str_array_length(input);
+
+	if (len < 4) {
+		if (error_r != NULL)
+			*error_r = "Corrupted data";
+		ret = FALSE;
+	} else if (*(input[0])== '1')
+		ret = dcrypt_openssl_load_private_key_dovecot_v1(key_r, len, input, password, key, error_r);
+	else if (*(input[0])== '2')
+		ret = dcrypt_openssl_load_private_key_dovecot_v2(key_r, len, input, password, key, error_r);
+	else {
+		if (error_r != NULL)
+			*error_r = "Unsupported key version";
+		ret = FALSE;
+	}
 	return ret;
 }
 
@@ -1254,23 +1251,21 @@ bool dcrypt_openssl_load_public_key_dovecot(struct dcrypt_public_key **key_r,
 {
 	int ec = 0;
 
-	T_BEGIN {
-		const char **input = t_strsplit_tab(data);
-		size_t len;
-		for(len=0;input[len]!=NULL;len++);
-		if (len < 2) ec = -1;
-		if (ec == 0 && *(input[0]) == '1') {
-			ec = dcrypt_openssl_load_public_key_dovecot_v1(key_r, len,
-				input, error_r);
-		} else if (ec == 0 && *(input[0]) == '2') {
-			ec = dcrypt_openssl_load_public_key_dovecot_v2(key_r, len,
-				input, error_r);
-		} else {
-			if (error_r != NULL)
-				*error_r = "Unsupported key version";
-			ec = -1;
-		}
-	} T_END;
+	const char **input = t_strsplit_tab(data);
+	size_t len = str_array_length(input);
+
+	if (len < 2) ec = -1;
+	if (ec == 0 && *(input[0]) == '1') {
+		ec = dcrypt_openssl_load_public_key_dovecot_v1(key_r, len,
+			input, error_r);
+	} else if (ec == 0 && *(input[0]) == '2') {
+		ec = dcrypt_openssl_load_public_key_dovecot_v2(key_r, len,
+			input, error_r);
+	} else {
+		if (error_r != NULL)
+			*error_r = "Unsupported key version";
+		ec = -1;
+	}
 
 	return (ec == 0 ? TRUE : FALSE);
 }
@@ -1426,16 +1421,9 @@ bool dcrypt_openssl_store_private_key_dovecot(struct dcrypt_private_key *key, co
 	}
 
 	/* append public key id */
-	struct dcrypt_public_key *pubkey = NULL;
-	if (!dcrypt_openssl_private_to_public_key(key, &pubkey, error_r)) {
-		buffer_set_used_size(destination, dest_used);
-		return FALSE;
-	}
-
 	str_append_c(destination, '\t');
 	buffer_set_used_size(buf, 0);
-	bool res = dcrypt_openssl_public_key_id(pubkey, "sha256", buf, error_r);
-	dcrypt_openssl_free_public_key(&pubkey);
+	bool res = dcrypt_openssl_private_key_id(key, "sha256", buf, error_r);
 	binary_to_hex_append(destination, buf->data, buf->used);
 
 	if (!res) {
@@ -1548,9 +1536,7 @@ bool dcrypt_openssl_store_private_key(struct dcrypt_private_key *key, enum dcryp
 	int ec;
 	if (format == DCRYPT_FORMAT_DOVECOT) {
 		bool ret;
-		T_BEGIN {
-			ret = dcrypt_openssl_store_private_key_dovecot(key, cipher, destination, password, enc_key, error_r);
-		} T_END;
+		ret = dcrypt_openssl_store_private_key_dovecot(key, cipher, destination, password, enc_key, error_r);
 		return ret;
 	}
 
@@ -1669,25 +1655,27 @@ bool dcrypt_openssl_key_string_get_info(const char *key_data, enum dcrypt_key_fo
 	}
 
 	/* is it PEM key */
-	if (strstr(key_data, "----- BEGIN ") != NULL) {
+	if (strncmp(key_data, "-----BEGIN ", 11) == 0) {
 		format = DCRYPT_FORMAT_PEM;
 		version = DCRYPT_KEY_VERSION_NA;
-		if (strstr(key_data, "ENCRYPTED") != NULL) {
+		key_data += 11;
+		if (strncmp(key_data, "ENCRYPTED ", 10) == 0) {
 			encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_PASSWORD;
+			key_data += 10;
 		}
-		if (strstr(key_data, "----- BEGIN PRIVATE KEY") != NULL)
+		if (strncmp(key_data, "PRIVATE KEY-----", 16) == 0)
 			kind = DCRYPT_KEY_KIND_PRIVATE;
-		else if (strstr(key_data, "----- BEGIN PUBLIC KEY") != NULL)
+		else if (strncmp(key_data, "PUBLIC KEY-----", 15) == 0)
 			kind = DCRYPT_KEY_KIND_PUBLIC;
 		else {
 			if (error_r != NULL)
 				*error_r = "Unknown/invalid PEM key type";
 			return FALSE;
 		}
-	} else T_BEGIN {
+	} else {
 		const char **fields = t_strsplit_tab(key_data);
-		int nfields;
-		for(nfields=0;fields[nfields]!=NULL;nfields++);
+		int nfields = str_array_length(fields);
+
 		if (nfields < 2) {
 			if (error_r != NULL)
 				*error_r = "Unknown key format";
@@ -1742,7 +1730,7 @@ bool dcrypt_openssl_key_string_get_info(const char *key_data, enum dcrypt_key_fo
 		/* last field is always key hash */
 		if (key_hash_r != NULL)
 			key_hash = i_strdup(fields[nfields-1]);
-	} T_END;
+	}
 
 	if (format_r != NULL) *format_r = format;
 	if (version_r != NULL) *version_r = version;
@@ -1907,30 +1895,43 @@ bool dcrypt_openssl_public_key_id_old(struct dcrypt_public_key *key, buffer_t *r
 	return TRUE;
 }
 
-/** this is the new which uses H(der formatted public key) **/
 static
-bool dcrypt_openssl_public_key_id(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r)
+bool dcrypt_openssl_private_key_id_old(struct dcrypt_private_key *key, buffer_t *result, const char **error_r)
 {
-	const EVP_MD *md = EVP_get_digestbyname(algorithm);
-	if (md == NULL) {
+	unsigned char buf[SHA256_DIGEST_LENGTH];
+	EVP_PKEY *priv = (EVP_PKEY*)key;
+
+	if (priv == NULL) {
 		if (error_r != NULL)
-			*error_r = t_strdup_printf("Unknown cipher %s", algorithm);
+			*error_r = "key is NULL";
 		return FALSE;
 	}
-	unsigned char buf[EVP_MD_size(md)];
-	EVP_PKEY *pub = (EVP_PKEY*)key;
-	const char *ptr;
-	bool res;
-	if (pub == NULL) {
+	if (EVP_PKEY_base_id(priv) != EVP_PKEY_EC) {
 		if (error_r != NULL)
-			*error_r = "key is NULL";
+			*error_r = "Only EC key supported";
 		return FALSE;
 	}
-	if (EVP_PKEY_base_id(pub) == EVP_PKEY_EC) {
-		EC_KEY_set_conv_form(EVP_PKEY_get0_EC_KEY(pub), POINT_CONVERSION_COMPRESSED);
+
+	char *pub_pt_hex = ec_key_get_pub_point_hex(EVP_PKEY_get0_EC_KEY(priv));
+	/* digest this */
+	SHA256((const unsigned char*)pub_pt_hex, strlen(pub_pt_hex), buf);
+	buffer_append(result, buf, SHA256_DIGEST_LENGTH);
+	OPENSSL_free(pub_pt_hex);
+	return TRUE;
+}
+
+/** this is the new which uses H(der formatted public key) **/
+static
+bool dcrypt_openssl_public_key_id_evp(EVP_PKEY *key, const EVP_MD *md, buffer_t *result, const char **error_r)
+{
+	bool res = FALSE;
+	unsigned char buf[EVP_MD_size(md)], *ptr;
+
+	if (EVP_PKEY_base_id(key) == EVP_PKEY_EC) {
+		EC_KEY_set_conv_form(EVP_PKEY_get0_EC_KEY(key), POINT_CONVERSION_COMPRESSED);
 	}
 	BIO *b = BIO_new(BIO_s_mem());
-	if (i2d_PUBKEY_bio(b, pub) < 1) {
+	if (i2d_PUBKEY_bio(b, key) < 1) {
 		BIO_vfree(b);
 		return dcrypt_openssl_error(error_r);
 	}
@@ -1950,7 +1951,6 @@ bool dcrypt_openssl_public_key_id(struct dcrypt_public_key *key, const char *alg
 		buffer_append(result, buf, hlen);
 		res = TRUE;
 	}
-
 #if SSLEAY_VERSION_NUMBER >= 0x1010000fL
 	EVP_MD_CTX_free(ctx);
 #else
@@ -1961,6 +1961,47 @@ bool dcrypt_openssl_public_key_id(struct dcrypt_public_key *key, const char *alg
 	return res;
 }
 
+static
+bool dcrypt_openssl_public_key_id(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r)
+{
+	const EVP_MD *md = EVP_get_digestbyname(algorithm);
+	EVP_PKEY *pub = (EVP_PKEY*)key;
+
+	if (md == NULL) {
+		if (error_r != NULL)
+			*error_r = t_strdup_printf("Unknown cipher %s", algorithm);
+		return FALSE;
+	}
+	if (pub == NULL) {
+		if (error_r != NULL)
+			*error_r = "key is NULL";
+		return FALSE;
+	}
+
+	return dcrypt_openssl_public_key_id_evp(pub, md, result, error_r);
+}
+
+static
+bool dcrypt_openssl_private_key_id(struct dcrypt_private_key *key, const char *algorithm, buffer_t *result, const char **error_r)
+{
+	const EVP_MD *md = EVP_get_digestbyname(algorithm);
+	EVP_PKEY *priv = (EVP_PKEY*)key;
+
+	if (md == NULL) {
+		if (error_r != NULL)
+			*error_r = t_strdup_printf("Unknown cipher %s", algorithm);
+		return FALSE;
+	}
+	if (priv == NULL) {
+		if (error_r != NULL)
+			*error_r = "key is NULL";
+		return FALSE;
+	}
+
+	return dcrypt_openssl_public_key_id_evp(priv, md, result, error_r);
+}
+
+
 static struct dcrypt_vfs dcrypt_openssl_vfs = {
 	.ctx_sym_create = dcrypt_openssl_ctx_sym_create,
 	.ctx_sym_destroy = dcrypt_openssl_ctx_sym_destroy,
@@ -2010,6 +2051,8 @@ static struct dcrypt_vfs dcrypt_openssl_vfs = {
 	.public_key_type = dcrypt_openssl_public_key_type,
 	.public_key_id = dcrypt_openssl_public_key_id,
 	.public_key_id_old = dcrypt_openssl_public_key_id_old,
+	.private_key_id = dcrypt_openssl_private_key_id,
+	.private_key_id_old = dcrypt_openssl_private_key_id_old,
 };
 
 void dcrypt_openssl_init(struct module *module ATTR_UNUSED)
diff --git a/src/lib-dcrypt/dcrypt-private.h b/src/lib-dcrypt/dcrypt-private.h
index 4936eb65f99..523b03b0cbb 100644
--- a/src/lib-dcrypt/dcrypt-private.h
+++ b/src/lib-dcrypt/dcrypt-private.h
@@ -92,6 +92,8 @@ struct dcrypt_vfs {
 	bool (*public_key_type)(struct dcrypt_public_key *key, enum dcrypt_key_type *key_type);
 	bool (*public_key_id)(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r);
 	bool (*public_key_id_old)(struct dcrypt_public_key *key, buffer_t *result, const char **error_r);
+	bool (*private_key_id)(struct dcrypt_private_key *key, const char *algorithm, buffer_t *result, const char **error_r);
+	bool (*private_key_id_old)(struct dcrypt_private_key *key, buffer_t *result, const char **error_r);
 };
 
 void dcrypt_set_vfs(struct dcrypt_vfs *vfs);
diff --git a/src/lib-dcrypt/dcrypt.c b/src/lib-dcrypt/dcrypt.c
index d4841993bbd..be35124c2b4 100644
--- a/src/lib-dcrypt/dcrypt.c
+++ b/src/lib-dcrypt/dcrypt.c
@@ -242,6 +242,14 @@ bool dcrypt_key_id_public_old(struct dcrypt_public_key *key, buffer_t *result, c
 {
 	return dcrypt_vfs->public_key_id_old(key, result, error_r);
 }
+bool dcrypt_key_id_private(struct dcrypt_private_key *key, const char *algorithm, buffer_t *result, const char **error_r)
+{
+	return dcrypt_vfs->private_key_id(key, algorithm, result, error_r);
+}
+bool dcrypt_key_id_private_old(struct dcrypt_private_key *key, buffer_t *result, const char **error_r)
+{
+	return dcrypt_vfs->private_key_id_old(key, result, error_r);
+}
 void dcrypt_keypair_free(struct dcrypt_keypair *keypair)
 {
 	dcrypt_vfs->free_keypair(keypair);
diff --git a/src/lib-dcrypt/dcrypt.h b/src/lib-dcrypt/dcrypt.h
index 07138d29a8f..70c4d896607 100644
--- a/src/lib-dcrypt/dcrypt.h
+++ b/src/lib-dcrypt/dcrypt.h
@@ -188,6 +188,8 @@ bool dcrypt_key_type_private(struct dcrypt_private_key *key, enum dcrypt_key_typ
 bool dcrypt_key_type_public(struct dcrypt_public_key *key, enum dcrypt_key_type *type);
 bool dcrypt_key_id_public(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r); /* return digest of key */
 bool dcrypt_key_id_public_old(struct dcrypt_public_key *key, buffer_t *result, const char **error_r); /* return SHA1 sum of key */
+bool dcrypt_key_id_private(struct dcrypt_private_key *key, const char *algorithm, buffer_t *result, const char **error_r); /* return digest of key */
+bool dcrypt_key_id_private_old(struct dcrypt_private_key *key, buffer_t *result, const char **error_r); /* return SHA1 sum of key */
 
 bool dcrypt_key_string_get_info(const char *key_data, enum dcrypt_key_format *format_r, enum dcrypt_key_version *version_r,
 	enum dcrypt_key_kind *kind_r, enum dcrypt_key_encryption_type *encryption_type_r, const char **encryption_key_hash_r,

From 77893565e6c9ac40915c29bd8455670171ce4464 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 12 Jun 2016 20:11:49 +0300
Subject: [PATCH 284/450] lib-index: mail_index_ext_resize() was broken when
 record_size wasn't changed.

---
 src/lib-index/mail-index-transaction-update.c | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/lib-index/mail-index-transaction-update.c b/src/lib-index/mail-index-transaction-update.c
index dff497e358f..784c7370b47 100644
--- a/src/lib-index/mail-index-transaction-update.c
+++ b/src/lib-index/mail-index-transaction-update.c
@@ -757,7 +757,6 @@ void mail_index_ext_resize(struct mail_index_transaction *t, uint32_t ext_id,
 	if (!mail_index_map_get_ext_idx(t->view->map, ext_id, &intro.ext_id)) {
 		/* have to create it */
 		intro.ext_id = (uint32_t)-1;
-		old_record_size = rext->record_size;
 		old_record_align = rext->record_align;
 		old_header_size = rext->hdr_size;
 	} else {
@@ -782,10 +781,10 @@ void mail_index_ext_resize(struct mail_index_transaction *t, uint32_t ext_id,
 		old_record_size = resizes[ext_id].record_size;
 	} else {
 		/* use the registered values. */
-		record_size = rext->record_size;
+		old_record_size = rext->record_size;
 	}
 
-	if (record_size != old_record_size) {
+	if (record_size != old_record_size && record_size != (uint16_t)-1) {
 		/* if record_size grows, we'll just resize the existing
 		   ext_rec_updates array. it's not possible to shrink
 		   record_size without data loss. */

From 2f83c2f73da136c31d52eac46a3a67f1ed8fa951 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martti=20Rannanj=C3=A4rvi?= <martti.rannanjarvi@dovecot.fi>
Date: Thu, 19 May 2016 11:14:46 +0300
Subject: [PATCH 285/450] master: make setting listen empty an error

---
 src/master/master-settings.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/master/master-settings.c b/src/master/master-settings.c
index da81dd075d3..216eb07be5e 100644
--- a/src/master/master-settings.c
+++ b/src/master/master-settings.c
@@ -432,6 +432,11 @@ master_settings_verify(void *_set, pool_t pool, const char **error_r)
 	unsigned int max_client_limit = set->default_client_limit;
 #endif
 
+	if (*set->listen == '\0') {
+		*error_r = "listen can't be set empty";
+		return FALSE;
+	}
+
 	len = strlen(set->base_dir);
 	if (len > 0 && set->base_dir[len-1] == '/') {
 		/* drop trailing '/' */

From dda8a314ba32ce28a7a4522857eae03aec31191e Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 13 Jun 2016 12:18:07 +0300
Subject: [PATCH 286/450] dovecot-config: Add lib-dcrypt to LIBDOVECOT_INCLUDE

Since lib-dcrypt is part of libdovecot.
---
 dovecot-config.in.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dovecot-config.in.in b/dovecot-config.in.in
index 8328de4d340..f776c387cf2 100644
--- a/dovecot-config.in.in
+++ b/dovecot-config.in.in
@@ -22,7 +22,7 @@ LIBDOVECOT_STORAGE_DEPS="@LIBDOVECOT_STORAGE_DEPS@"
 LIBDOVECOT_DSYNC_DEPS="@LIBDOVECOT_DSYNC@"
 LIBDOVECOT_LIBFTS_DEPS="@LIBDOVECOT_LIBFTS@"
 
-LIBDOVECOT_INCLUDE="-I$(incdir) -I$(incdir)/src/lib -I$(incdir)/src/lib-dict -I$(incdir)/src/lib-dns -I$(incdir)/src/lib-http -I$(incdir)/src/lib-mail -I$(incdir)/src/lib-imap -I$(incdir)/src/lib-fs -I$(incdir)/src/lib-charset -I$(incdir)/src/lib-auth -I$(incdir)/src/lib-master -I$(incdir)/src/lib-ssl-iostream -I$(incdir)/src/lib-compression -I$(incdir)/src/lib-settings -I$(incdir)/src/lib-test -I$(incdir)/src/lib-sasl -I$(incdir)/src/lib-stats"
+LIBDOVECOT_INCLUDE="-I$(incdir) -I$(incdir)/src/lib -I$(incdir)/src/lib-dict -I$(incdir)/src/lib-dns -I$(incdir)/src/lib-http -I$(incdir)/src/lib-mail -I$(incdir)/src/lib-imap -I$(incdir)/src/lib-fs -I$(incdir)/src/lib-charset -I$(incdir)/src/lib-auth -I$(incdir)/src/lib-master -I$(incdir)/src/lib-ssl-iostream -I$(incdir)/src/lib-compression -I$(incdir)/src/lib-settings -I$(incdir)/src/lib-test -I$(incdir)/src/lib-sasl -I$(incdir)/src/lib-stats -I$(incdir)/src/lib-dcrypt"
 LIBDOVECOT_LDA_INCLUDE="-I$(incdir)/src/lib-lda -I$(incdir)/src/lda"
 LIBDOVECOT_AUTH_INCLUDE="-I$(incdir)/src/auth"
 LIBDOVECOT_DOVEADM_INCLUDE="-I$(incdir)/src/doveadm"

From cecf8cfdc897d6a35629d8ff4efbd0d4638d7836 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Wed, 11 May 2016 08:02:29 +0300
Subject: [PATCH 287/450] ldap: Fix cyclic dependency

---
 configure.ac                                  |  1 +
 src/Makefile.am                               |  2 +-
 src/lib-dict-extra/Makefile.am                | 17 ---------------
 src/plugins/Makefile.am                       |  5 +++++
 src/plugins/dict-ldap/Makefile.am             | 21 +++++++++++++++++++
 .../dict-ldap}/dict-ldap-settings.c           |  0
 .../dict-ldap}/dict-ldap-settings.h           |  0
 .../dict-ldap}/dict-ldap.c                    |  0
 8 files changed, 28 insertions(+), 18 deletions(-)
 create mode 100644 src/plugins/dict-ldap/Makefile.am
 rename src/{lib-dict-extra => plugins/dict-ldap}/dict-ldap-settings.c (100%)
 rename src/{lib-dict-extra => plugins/dict-ldap}/dict-ldap-settings.h (100%)
 rename src/{lib-dict-extra => plugins/dict-ldap}/dict-ldap.c (100%)

diff --git a/configure.ac b/configure.ac
index 791c960f30b..060ab3acb36 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2929,6 +2929,7 @@ src/plugins/Makefile
 src/plugins/acl/Makefile
 src/plugins/imap-acl/Makefile
 src/plugins/autocreate/Makefile
+src/plugins/dict-ldap/Makefile
 src/plugins/expire/Makefile
 src/plugins/fs-compress/Makefile
 src/plugins/fts/Makefile
diff --git a/src/Makefile.am b/src/Makefile.am
index b456be5db5a..086b7c8fb5b 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -23,9 +23,9 @@ LIBDOVECOT_SUBDIRS = \
 
 SUBDIRS = \
 	$(LIBDOVECOT_SUBDIRS) \
-	$(LIB_LDAP) \
 	lib-dict-extra \
 	lib-dovecot \
+	$(LIB_LDAP) \
 	lib-fts \
 	lib-imap-client \
 	lib-imap-urlauth \
diff --git a/src/lib-dict-extra/Makefile.am b/src/lib-dict-extra/Makefile.am
index 7cf50114a06..e39d112ad8d 100644
--- a/src/lib-dict-extra/Makefile.am
+++ b/src/lib-dict-extra/Makefile.am
@@ -6,7 +6,6 @@ AM_CPPFLAGS = \
 	-I$(top_srcdir)/src/lib \
 	-I$(top_srcdir)/src/lib-dict \
 	-I$(top_srcdir)/src/lib-fs \
-	-I$(top_srcdir)/src/lib-ldap \
 	-I$(top_srcdir)/src/lib-settings
 
 libdict_extra_la_SOURCES = \
@@ -14,19 +13,3 @@ libdict_extra_la_SOURCES = \
 	dict-register.c
 
 NOPLUGIN_LDFLAGS =
-
-if HAVE_LDAP
-LIBDICT_LDAP = libdict_ldap.la
-endif
-libdict_ldap_la_LDFLAGS = -module -avoid-version $(LIBDOVECOT_LDAP)
-
-module_dictdir = $(moduledir)/dict
-module_dict_LTLIBRARIES = \
-	$(LIBDICT_LDAP)
-
-libdict_ldap_la_SOURCES = \
-	dict-ldap.c \
-	dict-ldap-settings.c
-
-noinst_HEADERS = \
-	dict-ldap-settings.h
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index b9196b38f73..7e0d00b17d9 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -10,6 +10,10 @@ if BUILD_SOLR
 FTS_SOLR = fts-solr
 endif
 
+if HAVE_LDAP
+DICT_LDAP = dict-ldap
+endif
+
 SUBDIRS = \
 	acl \
 	imap-acl \
@@ -38,4 +42,5 @@ SUBDIRS = \
 	$(ZLIB) \
 	$(FTS_LUCENE) \
 	$(FTS_SOLR) \
+	$(DICT_LDAP) \
 	fs-compress
diff --git a/src/plugins/dict-ldap/Makefile.am b/src/plugins/dict-ldap/Makefile.am
new file mode 100644
index 00000000000..7caeba67c91
--- /dev/null
+++ b/src/plugins/dict-ldap/Makefile.am
@@ -0,0 +1,21 @@
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-dict \
+	-I$(top_srcdir)/src/lib-ldap \
+	-I$(top_srcdir)/src/lib-settings
+
+LIBDICT_LDAP = libdict_ldap.la
+libdict_ldap_la_DEPENDENCIES = $(LIBDOVECOT_LDAP)
+libdict_ldap_la_LDFLAGS = -module -avoid-version
+libdict_ldap_la_LIBADD = $(LIBDOVECOT_LDAP)
+
+module_dictdir = $(moduledir)/dict
+module_dict_LTLIBRARIES = \
+	$(LIBDICT_LDAP)
+
+libdict_ldap_la_SOURCES = \
+	dict-ldap.c \
+	dict-ldap-settings.c
+
+noinst_HEADERS = \
+	dict-ldap-settings.h
diff --git a/src/lib-dict-extra/dict-ldap-settings.c b/src/plugins/dict-ldap/dict-ldap-settings.c
similarity index 100%
rename from src/lib-dict-extra/dict-ldap-settings.c
rename to src/plugins/dict-ldap/dict-ldap-settings.c
diff --git a/src/lib-dict-extra/dict-ldap-settings.h b/src/plugins/dict-ldap/dict-ldap-settings.h
similarity index 100%
rename from src/lib-dict-extra/dict-ldap-settings.h
rename to src/plugins/dict-ldap/dict-ldap-settings.h
diff --git a/src/lib-dict-extra/dict-ldap.c b/src/plugins/dict-ldap/dict-ldap.c
similarity index 100%
rename from src/lib-dict-extra/dict-ldap.c
rename to src/plugins/dict-ldap/dict-ldap.c

From 3f7937766d460f336253b3a9c597d82d959ff35a Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Mon, 13 Jun 2016 12:44:09 +0300
Subject: [PATCH 288/450] lib-dcrypt: Use dcrypt_key_id_private when applicable

---
 src/lib-dcrypt/istream-decrypt.c | 20 ++++----------------
 1 file changed, 4 insertions(+), 16 deletions(-)

diff --git a/src/lib-dcrypt/istream-decrypt.c b/src/lib-dcrypt/istream-decrypt.c
index d8c6329aaa9..77c68c33819 100644
--- a/src/lib-dcrypt/istream-decrypt.c
+++ b/src/lib-dcrypt/istream-decrypt.c
@@ -138,25 +138,16 @@ ssize_t i_stream_decrypt_read_header_v1(struct decrypt_istream *stream,
 	}
 
 	buffer_t *check = buffer_create_dynamic(pool_datastack_create(), 32);
-	struct dcrypt_public_key *pubkey = NULL;
 
-	/* do we have correct private key? */
-	if (!dcrypt_key_convert_private_to_public(stream->priv_key, &pubkey, &error)) {
-		io_stream_set_error(&stream->istream.iostream, "Cannot convert private key to public: %s", error);
-		return -1;
-	}
-	ec = 0;
-	if (!dcrypt_key_id_public_old(pubkey, check, &error)) {
+	if (!dcrypt_key_id_private_old(stream->priv_key, check, &error)) {
 		io_stream_set_error(&stream->istream.iostream, "Cannot get public key hash: %s", error);
-		ec = -1;
+		return -1;
 	} else {
 		if (memcmp(digest_pos, check->data, I_MIN(digest_len,check->used)) != 0) {
 			io_stream_set_error(&stream->istream.iostream, "Private key not available");
-			ec = -1;
+			return -1;
 		}
 	}
-	dcrypt_key_free_public(&pubkey);
-	if (ec != 0) return ec;
 
 	/* derive shared secret */
 	if (!dcrypt_ecdh_derive_secret_local(stream->priv_key, &ephemeral_key, secret, &error)) {
@@ -296,10 +287,7 @@ ssize_t i_stream_decrypt_key(struct decrypt_istream *stream, const char *malg, u
 			return -1;
 		}
 		buffer_create_from_data(&buf, dgst, sizeof(dgst));
-		struct dcrypt_public_key *pub = NULL;
-		dcrypt_key_convert_private_to_public(stream->priv_key, &pub, NULL);
-		dcrypt_key_id_public(pub, "sha256", &buf, NULL);
-		dcrypt_key_free_public(&pub);
+		dcrypt_key_id_private(stream->priv_key, "sha256", &buf, NULL);
 	}
 
 	/* for each key */

From dce4c1d98e4e529eb452989c4238989280dfad47 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 7 Jun 2016 03:47:44 +0300
Subject: [PATCH 289/450] Fixed linking with OSX.

---
 src/lib-dcrypt/Makefile.am | 5 +++--
 src/lib-ldap/Makefile.am   | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/lib-dcrypt/Makefile.am b/src/lib-dcrypt/Makefile.am
index 1ea6f74d21d..ea75e0c1f92 100644
--- a/src/lib-dcrypt/Makefile.am
+++ b/src/lib-dcrypt/Makefile.am
@@ -15,9 +15,10 @@ libdcrypt_la_CFLAGS = $(AM_CPPFLAGS) \
 
 if BUILD_OPENSSL
 pkglib_LTLIBRARIES += libdcrypt_openssl.la
-libdcrypt_openssl_la_SOURCES = dcrypt-openssl.c
-libdcrypt_openssl_la_LDFLAGS = -module -avoid-version -shared $(SSL_LIBS)
+libdcrypt_openssl_la_SOURCES = dcrypt-openssl.c dcrypt.c
+libdcrypt_openssl_la_LDFLAGS = -module -avoid-version -shared $(SSL_LIBS) ../lib/liblib.la
 libdcrypt_openssl_la_CFLAGS = $(AM_CPPFLAGS) \
+	-DDCRYPT_MODULE_DIR=\"$(pkglibdir)\"
 	$(SSL_CFLAGS)
 endif
 
diff --git a/src/lib-ldap/Makefile.am b/src/lib-ldap/Makefile.am
index b990e93fdf3..5407d1a6868 100644
--- a/src/lib-ldap/Makefile.am
+++ b/src/lib-ldap/Makefile.am
@@ -17,9 +17,9 @@ libdovecot_ldap_la_SOURCES = \
 	ldap-compare.c \
 	ldap-entry.c
 
-libdovecot_ldap_la_DEPENDENCIES =
+libdovecot_ldap_la_DEPENDENCIES = ../lib/liblib.la
 libdovecot_ldap_la_LDFLAGS = -export-dynamic
-libdovecot_ldap_la_LIBADD = $(LDAP_LIBS)
+libdovecot_ldap_la_LIBADD = ../lib/liblib.la $(LDAP_LIBS)
 
 headers = \
 	ldap-client.h

From 07a0a1869e5bbd070117a0c065b2ca70908cf6d2 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Mon, 13 Jun 2016 12:49:19 +0300
Subject: [PATCH 290/450] lib-dcrypt: Use hex encoded public key ID in callback

---
 src/lib-dcrypt/istream-decrypt.c | 7 +++----
 src/lib-dcrypt/istream-decrypt.h | 2 +-
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/lib-dcrypt/istream-decrypt.c b/src/lib-dcrypt/istream-decrypt.c
index 77c68c33819..87b954dd70d 100644
--- a/src/lib-dcrypt/istream-decrypt.c
+++ b/src/lib-dcrypt/istream-decrypt.c
@@ -120,8 +120,7 @@ ssize_t i_stream_decrypt_read_header_v1(struct decrypt_istream *stream,
 	if (stream->priv_key == NULL) {
 		/* see if we can get one */
 		if (stream->key_callback != NULL) {
-			unsigned char *key_id = t_malloc(digest_len);
-			memcpy(key_id, digest_pos, digest_len);
+			const char *key_id = binary_to_hex(digest_pos, digest_len);
 			int ret = stream->key_callback(key_id, &(stream->priv_key), &error, stream->key_context);
 			if (ret < 0) {
 				io_stream_set_error(&stream->istream.iostream, "Private key not available: %s", error);
@@ -297,9 +296,9 @@ ssize_t i_stream_decrypt_key(struct decrypt_istream *stream, const char *malg, u
 		ktype = *data++;
 
 		if (stream->key_callback != NULL) {
-			memcpy(dgst, data, sizeof(dgst));
+			const char *hexdgst = binary_to_hex(data, sizeof(dgst)); /* digest length */
 			/* hope you going to give us right key.. */
-			int ret = stream->key_callback(dgst, &(stream->priv_key), &error, stream->key_context);
+			int ret = stream->key_callback(hexdgst, &(stream->priv_key), &error, stream->key_context);
 			if (ret < 0) {
 				io_stream_set_error(&stream->istream.iostream, "Private key not available: %s", error);
 				return -1;
diff --git a/src/lib-dcrypt/istream-decrypt.h b/src/lib-dcrypt/istream-decrypt.h
index 713afbbcd42..0c59bc8ee82 100644
--- a/src/lib-dcrypt/istream-decrypt.h
+++ b/src/lib-dcrypt/istream-decrypt.h
@@ -7,7 +7,7 @@ struct dcrypt_context_symmetric;
 /* Look for a private key for a specified public key digest and set it to
    priv_key_r. Returns 1 if ok, 0 if key doesn't exist, -1 on internal error. */
 typedef int
-i_stream_decrypt_get_key_callback_t(const unsigned char *pubkey_digest,
+i_stream_decrypt_get_key_callback_t(const char *pubkey_digest,
 				    struct dcrypt_private_key **priv_key_r,
 				    const char **error_r, void *context);
 

From 3986d36de23a48beb85ff5ed5e3a864cb7ea9b12 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 13 Jun 2016 17:31:34 +0300
Subject: [PATCH 291/450] lib-fs: Code cleanup for fs-sis: Use struct fs.parent

---
 src/lib-fs/fs-sis-queue.c | 27 ++++++++++-----------------
 src/lib-fs/fs-sis.c       | 28 ++++++++++------------------
 2 files changed, 20 insertions(+), 35 deletions(-)

diff --git a/src/lib-fs/fs-sis-queue.c b/src/lib-fs/fs-sis-queue.c
index 9b330229d0c..fe970d84704 100644
--- a/src/lib-fs/fs-sis-queue.c
+++ b/src/lib-fs/fs-sis-queue.c
@@ -10,7 +10,6 @@
 
 struct sis_queue_fs {
 	struct fs fs;
-	struct fs *super;
 	char *queue_dir;
 };
 
@@ -22,7 +21,7 @@ struct sis_queue_fs_file {
 
 static void fs_sis_queue_copy_error(struct sis_queue_fs *fs)
 {
-	fs_set_error(&fs->fs, "%s", fs_last_error(fs->super));
+	fs_set_error(&fs->fs, "%s", fs_last_error(fs->fs.parent));
 }
 
 static void fs_sis_queue_file_copy_error(struct sis_queue_fs_file *file)
@@ -64,7 +63,7 @@ fs_sis_queue_init(struct fs *_fs, const char *args,
 		parent_args = "";
 	else
 		parent_name = t_strdup_until(parent_name, parent_args++);
-	if (fs_init(parent_name, parent_args, set, &fs->super, &error) < 0) {
+	if (fs_init(parent_name, parent_args, set, &_fs->parent, &error) < 0) {
 		fs_set_error(_fs, "%s", error);
 		return -1;
 	}
@@ -75,17 +74,15 @@ static void fs_sis_queue_deinit(struct fs *_fs)
 {
 	struct sis_queue_fs *fs = (struct sis_queue_fs *)_fs;
 
-	if (fs->super != NULL)
-		fs_deinit(&fs->super);
+	if (_fs->parent != NULL)
+		fs_deinit(&_fs->parent);
 	i_free(fs->queue_dir);
 	i_free(fs);
 }
 
 static enum fs_properties fs_sis_queue_get_properties(struct fs *_fs)
 {
-	struct sis_queue_fs *fs = (struct sis_queue_fs *)_fs;
-
-	return fs_get_properties(fs->super);
+	return fs_get_properties(_fs->parent);
 }
 
 static struct fs_file *
@@ -103,7 +100,7 @@ fs_sis_queue_file_init(struct fs *_fs, const char *path,
 	if (mode == FS_OPEN_MODE_APPEND)
 		fs_set_error(_fs, "APPEND mode not supported");
 	else
-		file->super = fs_file_init(fs->super, path, mode | flags);
+		file->super = fs_file_init(_fs->parent, path, mode | flags);
 	return &file->file;
 }
 
@@ -144,9 +141,7 @@ fs_sis_queue_set_async_callback(struct fs_file *_file,
 
 static int fs_sis_queue_wait_async(struct fs *_fs)
 {
-	struct sis_queue_fs *fs = (struct sis_queue_fs *)_fs;
-
-	return fs_wait_async(fs->super);
+	return fs_wait_async(_fs->parent);
 }
 
 static void
@@ -206,9 +201,9 @@ static void fs_sis_queue_add(struct sis_queue_fs_file *file)
 		fname = path;
 
 	queue_path = t_strdup_printf("%s/%s", fs->queue_dir, fname);
-	queue_file = fs_file_init(fs->super, queue_path, FS_OPEN_MODE_CREATE);
+	queue_file = fs_file_init(fs->fs.parent, queue_path, FS_OPEN_MODE_CREATE);
 	if (fs_write(queue_file, "", 0) < 0 && errno != EEXIST)
-		i_error("fs-sis-queue: %s", fs_last_error(fs->super));
+		i_error("fs-sis-queue: %s", fs_last_error(fs->fs.parent));
 	fs_file_deinit(&queue_file);
 }
 
@@ -348,9 +343,7 @@ static struct fs_iter *
 fs_sis_queue_iter_init(struct fs *_fs, const char *path,
 		       enum fs_iter_flags flags)
 {
-	struct sis_queue_fs *fs = (struct sis_queue_fs *)_fs;
-
-	return fs_iter_init(fs->super, path, flags);
+	return fs_iter_init(_fs->parent, path, flags);
 }
 
 const struct fs fs_class_sis_queue = {
diff --git a/src/lib-fs/fs-sis.c b/src/lib-fs/fs-sis.c
index 1563c29592a..8c705f171cf 100644
--- a/src/lib-fs/fs-sis.c
+++ b/src/lib-fs/fs-sis.c
@@ -12,7 +12,6 @@
 
 struct sis_fs {
 	struct fs fs;
-	struct fs *super;
 };
 
 struct sis_fs_file {
@@ -31,7 +30,7 @@ struct sis_fs_file {
 
 static void fs_sis_copy_error(struct sis_fs *fs)
 {
-	fs_set_error(&fs->fs, "%s", fs_last_error(fs->super));
+	fs_set_error(&fs->fs, "%s", fs_last_error(fs->fs.parent));
 }
 
 static void fs_sis_file_copy_error(struct sis_fs_file *file)
@@ -53,7 +52,6 @@ static struct fs *fs_sis_alloc(void)
 static int
 fs_sis_init(struct fs *_fs, const char *args, const struct fs_settings *set)
 {
-	struct sis_fs *fs = (struct sis_fs *)_fs;
 	enum fs_properties props;
 	const char *parent_name, *parent_args, *error;
 
@@ -70,11 +68,11 @@ fs_sis_init(struct fs *_fs, const char *args, const struct fs_settings *set)
 		parent_name = t_strdup_until(args, parent_args);
 		parent_args++;
 	}
-	if (fs_init(parent_name, parent_args, set, &fs->super, &error) < 0) {
+	if (fs_init(parent_name, parent_args, set, &_fs->parent, &error) < 0) {
 		fs_set_error(_fs, "%s", error);
 		return -1;
 	}
-	props = fs_get_properties(fs->super);
+	props = fs_get_properties(_fs->parent);
 	if ((props & FS_SIS_REQUIRED_PROPS) != FS_SIS_REQUIRED_PROPS) {
 		fs_set_error(_fs, "%s backend can't be used with SIS",
 			     parent_name);
@@ -87,16 +85,14 @@ static void fs_sis_deinit(struct fs *_fs)
 {
 	struct sis_fs *fs = (struct sis_fs *)_fs;
 
-	if (fs->super != NULL)
-		fs_deinit(&fs->super);
+	if (_fs->parent != NULL)
+		fs_deinit(&_fs->parent);
 	i_free(fs);
 }
 
 static enum fs_properties fs_sis_get_properties(struct fs *_fs)
 {
-	struct sis_fs *fs = (struct sis_fs *)_fs;
-
-	return fs_get_properties(fs->super);
+	return fs_get_properties(_fs->parent);
 }
 
 static struct fs_file *
@@ -124,7 +120,7 @@ fs_sis_file_init(struct fs *_fs, const char *path,
 
 	/* if hashes/<hash> already exists, open it */
 	file->hash_path = i_strdup_printf("%s/"HASH_DIR_NAME"/%s", dir, hash);
-	file->hash_file = fs_file_init(fs->super, file->hash_path,
+	file->hash_file = fs_file_init(_fs->parent, file->hash_path,
 				       FS_OPEN_MODE_READONLY);
 
 	file->hash_input = fs_read_stream(file->hash_file, IO_BLOCK_SIZE);
@@ -137,7 +133,7 @@ fs_sis_file_init(struct fs *_fs, const char *path,
 		i_stream_destroy(&file->hash_input);
 	}
 
-	file->super = fs_file_init(fs->super, path, mode | flags);
+	file->super = fs_file_init(_fs->parent, path, mode | flags);
 	return &file->file;
 }
 
@@ -181,9 +177,7 @@ fs_sis_set_async_callback(struct fs_file *_file,
 
 static int fs_sis_wait_async(struct fs *_fs)
 {
-	struct sis_fs *fs = (struct sis_fs *)_fs;
-
-	return fs_wait_async(fs->super);
+	return fs_wait_async(_fs->parent);
 }
 
 static void
@@ -486,9 +480,7 @@ static int fs_sis_delete(struct fs_file *_file)
 static struct fs_iter *
 fs_sis_iter_init(struct fs *_fs, const char *path, enum fs_iter_flags flags)
 {
-	struct sis_fs *fs = (struct sis_fs *)_fs;
-
-	return fs_iter_init(fs->super, path, flags);
+	return fs_iter_init(_fs->parent, path, flags);
 }
 
 const struct fs fs_class_sis = {

From 7e55f1ea2fd9610c500178bf6d5401bf88648818 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 13 Jun 2016 17:11:28 +0300
Subject: [PATCH 292/450] lib-fs: Added fs_switch_ioloop()

---
 src/lib-fs/fs-api-private.h           |  2 ++
 src/lib-fs/fs-api.c                   | 14 ++++++++++++++
 src/lib-fs/fs-api.h                   |  4 ++++
 src/lib-fs/fs-dict.c                  |  3 ++-
 src/lib-fs/fs-metawrap.c              |  3 ++-
 src/lib-fs/fs-posix.c                 |  3 ++-
 src/lib-fs/fs-randomfail.c            |  3 ++-
 src/lib-fs/fs-sis-queue.c             |  1 +
 src/lib-fs/fs-sis.c                   |  1 +
 src/lib-fs/fs-test.c                  |  3 ++-
 src/plugins/fs-compress/fs-compress.c |  3 ++-
 11 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/src/lib-fs/fs-api-private.h b/src/lib-fs/fs-api-private.h
index 4b480adb361..fa5168b575a 100644
--- a/src/lib-fs/fs-api-private.h
+++ b/src/lib-fs/fs-api-private.h
@@ -66,6 +66,8 @@ struct fs_vfuncs {
 				     enum fs_iter_flags flags);
 	const char *(*iter_next)(struct fs_iter *iter);
 	int (*iter_deinit)(struct fs_iter *iter);
+
+	bool (*switch_ioloop)(struct fs *fs);
 };
 
 struct fs {
diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c
index 024cec4ad62..dc892d2014f 100644
--- a/src/lib-fs/fs-api.c
+++ b/src/lib-fs/fs-api.c
@@ -737,6 +737,20 @@ int fs_wait_async(struct fs *fs)
 	return ret;
 }
 
+bool fs_switch_ioloop(struct fs *fs)
+{
+	bool ret = FALSE;
+
+	if (fs->v.switch_ioloop != NULL) {
+		T_BEGIN {
+			ret = fs->v.switch_ioloop(fs);
+		} T_END;
+	} else if (fs->parent != NULL) {
+		ret = fs_switch_ioloop(fs->parent);
+	}
+	return ret;
+}
+
 int fs_lock(struct fs_file *file, unsigned int secs, struct fs_lock **lock_r)
 {
 	int ret;
diff --git a/src/lib-fs/fs-api.h b/src/lib-fs/fs-api.h
index d5702533f87..2f8da389cc8 100644
--- a/src/lib-fs/fs-api.h
+++ b/src/lib-fs/fs-api.h
@@ -288,6 +288,10 @@ void fs_file_set_async_callback(struct fs_file *file,
    It's an error to call this when there are no pending async operations.
    Returns 0 if ok, -1 if timed out. */
 int fs_wait_async(struct fs *fs);
+/* Switch the fs to the current ioloop. This can be used to do fs_wait_async()
+   among other IO work. Returns TRUE if there is actually some work that can
+   be waited on. */
+bool fs_switch_ioloop(struct fs *fs) ATTR_NOWARN_UNUSED_RESULT;
 
 /* Returns 1 if file exists, 0 if not, -1 if error occurred. */
 int fs_exists(struct fs_file *file);
diff --git a/src/lib-fs/fs-dict.c b/src/lib-fs/fs-dict.c
index 27049502988..5efd33abb65 100644
--- a/src/lib-fs/fs-dict.c
+++ b/src/lib-fs/fs-dict.c
@@ -318,6 +318,7 @@ const struct fs fs_class_dict = {
 		fs_dict_delete,
 		fs_dict_iter_init,
 		fs_dict_iter_next,
-		fs_dict_iter_deinit
+		fs_dict_iter_deinit,
+		NULL
 	}
 };
diff --git a/src/lib-fs/fs-metawrap.c b/src/lib-fs/fs-metawrap.c
index c4887d75c25..85b2f059a03 100644
--- a/src/lib-fs/fs-metawrap.c
+++ b/src/lib-fs/fs-metawrap.c
@@ -613,6 +613,7 @@ const struct fs fs_class_metawrap = {
 		fs_metawrap_delete,
 		fs_metawrap_iter_init,
 		fs_metawrap_iter_next,
-		fs_metawrap_iter_deinit
+		fs_metawrap_iter_deinit,
+		NULL
 	}
 };
diff --git a/src/lib-fs/fs-posix.c b/src/lib-fs/fs-posix.c
index ddd9ccb6871..de9f43ead75 100644
--- a/src/lib-fs/fs-posix.c
+++ b/src/lib-fs/fs-posix.c
@@ -873,6 +873,7 @@ const struct fs fs_class_posix = {
 		fs_posix_delete,
 		fs_posix_iter_init,
 		fs_posix_iter_next,
-		fs_posix_iter_deinit
+		fs_posix_iter_deinit,
+		NULL
 	}
 };
diff --git a/src/lib-fs/fs-randomfail.c b/src/lib-fs/fs-randomfail.c
index 3a397dec996..ff4e80f76ee 100644
--- a/src/lib-fs/fs-randomfail.c
+++ b/src/lib-fs/fs-randomfail.c
@@ -567,6 +567,7 @@ const struct fs fs_class_randomfail = {
 		fs_randomfail_delete,
 		fs_randomfail_iter_init,
 		fs_randomfail_iter_next,
-		fs_randomfail_iter_deinit
+		fs_randomfail_iter_deinit,
+		NULL
 	}
 };
diff --git a/src/lib-fs/fs-sis-queue.c b/src/lib-fs/fs-sis-queue.c
index fe970d84704..eaefb50e0ba 100644
--- a/src/lib-fs/fs-sis-queue.c
+++ b/src/lib-fs/fs-sis-queue.c
@@ -376,6 +376,7 @@ const struct fs fs_class_sis_queue = {
 		fs_sis_queue_delete,
 		fs_sis_queue_iter_init,
 		NULL,
+		NULL,
 		NULL
 	}
 };
diff --git a/src/lib-fs/fs-sis.c b/src/lib-fs/fs-sis.c
index 8c705f171cf..98ccf96b16b 100644
--- a/src/lib-fs/fs-sis.c
+++ b/src/lib-fs/fs-sis.c
@@ -513,6 +513,7 @@ const struct fs fs_class_sis = {
 		fs_sis_delete,
 		fs_sis_iter_init,
 		NULL,
+		NULL,
 		NULL
 	}
 };
diff --git a/src/lib-fs/fs-test.c b/src/lib-fs/fs-test.c
index e0627797df9..b3c53a65f95 100644
--- a/src/lib-fs/fs-test.c
+++ b/src/lib-fs/fs-test.c
@@ -417,6 +417,7 @@ const struct fs fs_class_test = {
 		fs_test_delete,
 		fs_test_iter_init,
 		fs_test_iter_next,
-		fs_test_iter_deinit
+		fs_test_iter_deinit,
+		NULL
 	}
 };
diff --git a/src/plugins/fs-compress/fs-compress.c b/src/plugins/fs-compress/fs-compress.c
index 4088c58db7f..81dfb8c66f8 100644
--- a/src/plugins/fs-compress/fs-compress.c
+++ b/src/plugins/fs-compress/fs-compress.c
@@ -413,6 +413,7 @@ const struct fs fs_class_compress = {
 		fs_compress_delete,
 		fs_compress_iter_init,
 		fs_compress_iter_next,
-		fs_compress_iter_deinit
+		fs_compress_iter_deinit,
+		NULL
 	}
 };

From 68df4591b8a902ef38975183353281f194c63377 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 13 Jun 2016 17:59:01 +0300
Subject: [PATCH 293/450] dict: Avoid potentially using 100% CPU

Continuing 65c570f18 fix.
---
 src/dict/dict-connection.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/dict/dict-connection.c b/src/dict/dict-connection.c
index 7484baa8012..226bf7fc888 100644
--- a/src/dict/dict-connection.c
+++ b/src/dict/dict-connection.c
@@ -109,6 +109,9 @@ static void dict_connection_input_more(struct dict_connection *conn)
 	const char *line;
 	int ret;
 
+	if (conn->to_input != NULL)
+		timeout_remove(&conn->to_input);
+
 	while ((line = i_stream_next_line(conn->input)) != NULL) {
 		T_BEGIN {
 			ret = dict_command_input(conn, line);
@@ -130,9 +133,6 @@ static void dict_connection_input(struct dict_connection *conn)
 {
 	const char *line;
 
-	if (conn->to_input != NULL)
-		timeout_remove(&conn->to_input);
-
 	switch (i_stream_read(conn->input)) {
 	case 0:
 		return;

From 6aed775268e17fda0a437e5da004c7b047fd4b17 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 13 Jun 2016 20:16:14 +0300
Subject: [PATCH 294/450] dict: Increased max number of pipelined requests from
 5 to 1000.

The client is supposed to be the one throttling the requests. We mainly
want to avoid accidental abuses. Using 1000 is hopefully "large enough"
without being "too large".
---
 src/dict/dict-connection.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/dict/dict-connection.c b/src/dict/dict-connection.c
index 226bf7fc888..4cbffc1d787 100644
--- a/src/dict/dict-connection.c
+++ b/src/dict/dict-connection.c
@@ -14,7 +14,7 @@
 
 #include <unistd.h>
 
-#define DICT_CONN_MAX_PENDING_COMMANDS 5
+#define DICT_CONN_MAX_PENDING_COMMANDS 1000
 
 static struct dict_connection *dict_connections;
 static unsigned int dict_connections_count = 0;

From acf2cc6f7f05fd609e835206ea5ae462c99c873f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 14 Jun 2016 01:14:13 +0300
Subject: [PATCH 295/450] lib-index: Fixes to handling resized records.

---
 src/lib-index/mail-index-sync-ext.c           | 20 ++++++++++++-------
 src/lib-index/mail-index-sync-private.h       |  1 +
 src/lib-index/mail-index-sync-update.c        |  5 +----
 src/lib-index/mail-index-transaction-export.c |  6 ++----
 4 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/src/lib-index/mail-index-sync-ext.c b/src/lib-index/mail-index-sync-ext.c
index e758e53df30..e6d475ba8b8 100644
--- a/src/lib-index/mail-index-sync-ext.c
+++ b/src/lib-index/mail-index-sync-ext.c
@@ -478,6 +478,7 @@ int mail_index_sync_ext_intro(struct mail_index_sync_map_ctx *ctx,
 	   intro is corrupted */
 	ctx->cur_ext_map_idx = (uint32_t)-2;
 	ctx->cur_ext_ignore = TRUE;
+	ctx->cur_ext_record_size = 0;
 
 	if (u->ext_id != (uint32_t)-1 &&
 	    (!array_is_created(&map->extensions) ||
@@ -537,6 +538,7 @@ int mail_index_sync_ext_intro(struct mail_index_sync_map_ctx *ctx,
 		return -1;
 	}
 
+	ctx->cur_ext_record_size = u->record_size;
 	if (ext != NULL) {
 		/* exists already */
 		if (u->reset_id == ext->reset_id) {
@@ -675,7 +677,7 @@ mail_index_sync_ext_rec_update(struct mail_index_sync_map_ctx *ctx,
 		return 1;
 
 	ext = array_idx(&view->map->extensions, ctx->cur_ext_map_idx);
-	i_assert(ext->record_offset + ext->record_size <=
+	i_assert(ext->record_offset + ctx->cur_ext_record_size <=
 		 view->map->hdr.record_size);
 
 	rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq);
@@ -696,7 +698,11 @@ mail_index_sync_ext_rec_update(struct mail_index_sync_map_ctx *ctx,
 	}
 
 	/* @UNSAFE */
-	memcpy(old_data, u + 1, ext->record_size);
+	memcpy(old_data, u + 1, ctx->cur_ext_record_size);
+	if (ctx->cur_ext_record_size < ext->record_size) {
+		memset(PTR_OFFSET(old_data, ctx->cur_ext_record_size), 0,
+		       ext->record_size - ctx->cur_ext_record_size);
+	}
 	return 1;
 }
 
@@ -724,7 +730,7 @@ mail_index_sync_ext_atomic_inc(struct mail_index_sync_map_ctx *ctx,
 		return 1;
 
 	ext = array_idx(&view->map->extensions, ctx->cur_ext_map_idx);
-	i_assert(ext->record_offset + ext->record_size <=
+	i_assert(ext->record_offset + ctx->cur_ext_record_size <=
 		 view->map->hdr.record_size);
 
 	rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq);
@@ -732,8 +738,8 @@ mail_index_sync_ext_atomic_inc(struct mail_index_sync_map_ctx *ctx,
 
 	min_value = u->diff >= 0 ? 0 : (uint64_t)(-(int64_t)u->diff);
 
-	max_value = ext->record_size == 8 ? (uint64_t)-1 :
-		((uint64_t)1 << (ext->record_size*8)) - 1;
+	max_value = ctx->cur_ext_record_size == 8 ? (uint64_t)-1 :
+		((uint64_t)1 << (ctx->cur_ext_record_size*8)) - 1;
 	if (u->diff <= 0) {
 		/* skip */
 	} else if (max_value >= (uint32_t)u->diff) {
@@ -745,7 +751,7 @@ mail_index_sync_ext_atomic_inc(struct mail_index_sync_map_ctx *ctx,
 		return -1;
 	}
 
-	switch (ext->record_size) {
+	switch (ctx->cur_ext_record_size) {
 	case 1: {
 		uint8_t *num = data;
 
@@ -778,7 +784,7 @@ mail_index_sync_ext_atomic_inc(struct mail_index_sync_map_ctx *ctx,
 	default:
 		mail_index_sync_set_corrupted(ctx,
 			"Extension record inc with invalid size=%u",
-			ext->record_size);
+			ctx->cur_ext_record_size);
 		return -1;
 	}
 	if (orig_num < min_value) {
diff --git a/src/lib-index/mail-index-sync-private.h b/src/lib-index/mail-index-sync-private.h
index 5ef8293208f..985c8da2b77 100644
--- a/src/lib-index/mail-index-sync-private.h
+++ b/src/lib-index/mail-index-sync-private.h
@@ -27,6 +27,7 @@ struct mail_index_sync_map_ctx {
 	struct mail_index_view *view;
 	struct mail_index_modseq_sync *modseq_ctx;
 	uint32_t cur_ext_map_idx;
+	uint32_t cur_ext_record_size;
 
 	uint32_t ext_intro_seq;
 	uoff_t ext_intro_offset, ext_intro_end_offset;
diff --git a/src/lib-index/mail-index-sync-update.c b/src/lib-index/mail-index-sync-update.c
index 0e0edee624f..05d33bc2bbc 100644
--- a/src/lib-index/mail-index-sync-update.c
+++ b/src/lib-index/mail-index-sync-update.c
@@ -712,7 +712,6 @@ mail_index_sync_record_real(struct mail_index_sync_map_ctx *ctx,
 	}
 	case MAIL_TRANSACTION_EXT_REC_UPDATE: {
 		const struct mail_transaction_ext_rec_update *rec;
-		const struct mail_index_ext *ext;
 		unsigned int i, record_size;
 
 		if (ctx->cur_ext_map_idx == (uint32_t)-1) {
@@ -728,10 +727,8 @@ mail_index_sync_record_real(struct mail_index_sync_map_ctx *ctx,
 			break;
 		}
 
-		ext = array_idx(&ctx->view->map->extensions,
-				ctx->cur_ext_map_idx);
 		/* the record is padded to 32bits in the transaction log */
-		record_size = (sizeof(*rec) + ext->record_size + 3) & ~3;
+		record_size = (sizeof(*rec) + ctx->cur_ext_record_size + 3) & ~3;
 
 		for (i = 0; i < hdr->size; i += record_size) {
 			rec = CONST_PTR_OFFSET(data, i);
diff --git a/src/lib-index/mail-index-transaction-export.c b/src/lib-index/mail-index-transaction-export.c
index 1996c80c900..f846c891f1c 100644
--- a/src/lib-index/mail-index-transaction-export.c
+++ b/src/lib-index/mail-index-transaction-export.c
@@ -138,16 +138,14 @@ static void log_append_ext_intro(struct mail_index_export_context *ctx,
 		/* generate a new intro structure */
 		intro = buffer_append_space_unsafe(buf, sizeof(*intro));
 		intro->ext_id = idx;
+		intro->record_size = rext->record_size;
+		intro->record_align = rext->record_align;
 		if (idx == (uint32_t)-1) {
 			intro->hdr_size = rext->hdr_size;
-			intro->record_size = rext->record_size;
-			intro->record_align = rext->record_align;
 			intro->name_size = strlen(rext->name);
 		} else {
 			ext = array_idx(&t->view->index->map->extensions, idx);
 			intro->hdr_size = ext->hdr_size;
-			intro->record_size = ext->record_size;
-			intro->record_align = ext->record_align;
 			intro->name_size = 0;
 		}
 		intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK;

From b4fe750eb72ad6428d44f1186f224dad0e9c898f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 14 Jun 2016 10:49:24 +0300
Subject: [PATCH 296/450] lib-index: Fixed test-mail-index-sync-ext

---
 src/lib-index/test-mail-index-sync-ext.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib-index/test-mail-index-sync-ext.c b/src/lib-index/test-mail-index-sync-ext.c
index 7a9f29d152e..b5c0a1170a7 100644
--- a/src/lib-index/test-mail-index-sync-ext.c
+++ b/src/lib-index/test-mail-index-sync-ext.c
@@ -63,7 +63,7 @@ static void test_mail_index_sync_ext_atomic_inc(void)
 	u.uid = 1;
 #define TEST_ATOMIC(_type, _value, _diff, _ret) \
 	{ _type *n = ptr; *n = _value; } \
-	ext->record_size = sizeof(_type); \
+	ctx.cur_ext_record_size = sizeof(_type); \
 	u.diff = _diff; \
 	test_assert(mail_index_sync_ext_atomic_inc(&ctx, &u) == _ret);
 
@@ -81,7 +81,7 @@ static void test_mail_index_sync_ext_atomic_inc(void)
 	TEST_ATOMIC_BLOCK(uint8_t, 255);
 	TEST_ATOMIC_BLOCK(uint16_t, 65535);
 
-	ext->record_size = 5;
+	ctx.cur_ext_record_size = 5;
 	u.diff = 0;
 	test_assert(mail_index_sync_ext_atomic_inc(&ctx, &u) == -1);
 

From 920a34e4eb5f6f8b16ae810442851b345180366c Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Tue, 14 Jun 2016 12:08:34 +0300
Subject: [PATCH 297/450] dcrypt-openssl: Pass pointer safely

---
 src/lib-dcrypt/dcrypt-openssl.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index 743c345c080..af230ee0282 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -1231,11 +1231,13 @@ int dcrypt_openssl_load_public_key_dovecot_v2(struct dcrypt_public_key **key_r,
 	buffer_t tmp;
 	size_t keylen = strlen(input[1])/2;
 	unsigned char keybuf[keylen];
+	const unsigned char *ptr;
 	buffer_create_from_data(&tmp, keybuf, keylen);
 	hex_to_binary(input[1], &tmp);
+	ptr = keybuf;
 
 	EVP_PKEY *pkey = EVP_PKEY_new();
-	if (d2i_PUBKEY(&pkey, (const unsigned char**)(tmp.data), tmp.used)==NULL) {
+	if (d2i_PUBKEY(&pkey, &ptr, keylen)==NULL) {
 		EVP_PKEY_free(pkey);
 		dcrypt_openssl_error(error_r);
 		return -1;

From 0901d3f50a25f605ad5fb0254f8457fdf093a747 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Tue, 14 Jun 2016 12:08:49 +0300
Subject: [PATCH 298/450] dcrypt: Add test for public key loading

---
 src/lib-dcrypt/test-crypto.c | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/src/lib-dcrypt/test-crypto.c b/src/lib-dcrypt/test-crypto.c
index 463e06562fb..d8432869e94 100644
--- a/src/lib-dcrypt/test-crypto.c
+++ b/src/lib-dcrypt/test-crypto.c
@@ -285,6 +285,26 @@ void test_load_v2_key(void)
 	test_end();
 }
 
+static
+void test_load_v2_public_key(void)
+{
+	struct dcrypt_public_key *pub;
+	const char *error;
+
+	test_begin("test_load_v2_public_key");
+	const char *key = "2\t305e301006072a8648ce3d020106052b81040026034a000303a9288126a4ef239199d7ebe784d0b81b545df40e1feac5980965914524005fd11d18cf71cfd875a037172275dda474bcf6a96fd4824c9019b108e5258c0548ee70c6ce1d67ca5d";
+
+	test_assert(dcrypt_key_load_public(&pub, DCRYPT_FORMAT_DOVECOT, key, &error));
+
+	buffer_t *tmp = buffer_create_dynamic(default_pool, 256);
+
+	test_assert(dcrypt_key_store_public(pub, DCRYPT_FORMAT_DOVECOT, tmp, &error));
+
+	test_assert(strcmp(key, str_c(tmp))==0);
+
+	test_end();
+}
+
 int main(void) {
 	dcrypt_initialize("openssl", NULL);
 	random_init();
@@ -294,6 +314,7 @@ int main(void) {
 		test_hmac_test_vectors,
 		test_load_v1_key,
 		test_load_v2_key,
+		test_load_v2_public_key,
 		NULL
 	};
 

From a63f6d227f200b1c73a3ebfc0b6131cd40d17f67 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Tue, 14 Jun 2016 12:30:27 +0300
Subject: [PATCH 299/450] dcrypt: Arm deinitialization code

---
 src/lib-dcrypt/dcrypt.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt.c b/src/lib-dcrypt/dcrypt.c
index be35124c2b4..e8b2d00bfe0 100644
--- a/src/lib-dcrypt/dcrypt.c
+++ b/src/lib-dcrypt/dcrypt.c
@@ -10,7 +10,9 @@ bool dcrypt_initialize(const char *backend, const char **error_r)
 {
 	struct module_dir_load_settings mod_set;
 
-	if (dcrypt_vfs != NULL) return TRUE;
+	if (dcrypt_vfs != NULL) {
+		return TRUE;
+	}
 	if (backend == NULL) backend = "openssl"; /* default for now */
 
 	const char *implementation = t_strconcat("dcrypt_",backend,NULL);
@@ -34,13 +36,12 @@ bool dcrypt_initialize(const char *backend, const char **error_r)
 	/* Destroy SSL module after(most of) the others. Especially lib-fs
 	   backends may still want to access SSL module in their own
 	   atexit-callbacks. */
-//	lib_atexit_priority(dcrypt_deinitialize, LIB_ATEXIT_PRIORITY_LOW);
+	lib_atexit_priority(dcrypt_deinitialize, LIB_ATEXIT_PRIORITY_LOW);
 	return TRUE;
 }
 
 void dcrypt_deinitialize(void)
 {
-	i_error("I got called");
 	if (dcrypt_module != NULL) {
 		dcrypt_module->deinit();
 		module_dir_unload(&dcrypt_module);

From cf936f221b84746644ff64d5d8c9b9208aa95452 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martti=20Rannanj=C3=A4rvi?= <martti.rannanjarvi@dovecot.fi>
Date: Tue, 10 May 2016 10:19:57 +0300
Subject: [PATCH 300/450] [LEN] to [static LEN] on some function parameters

Also add STATIC_ARRAY macro to hide it in c++ compilation.
---
 src/auth/mech-rpa.c                                | 4 ++--
 src/auth/passdb.c                                  | 2 +-
 src/auth/passdb.h                                  | 2 +-
 src/auth/userdb.c                                  | 2 +-
 src/auth/userdb.h                                  | 2 +-
 src/lib-storage/index/mbox/mbox-md5-all.c          | 2 +-
 src/lib-storage/index/mbox/mbox-md5-apop3d.c       | 2 +-
 src/lib-storage/index/mbox/mbox-md5.h              | 2 +-
 src/lib/guid.c                                     | 2 +-
 src/lib/guid.h                                     | 2 +-
 src/lib/macros.h                                   | 6 ++++++
 src/lib/md4.c                                      | 4 ++--
 src/lib/md4.h                                      | 5 +++--
 src/lib/md5.c                                      | 4 ++--
 src/lib/md5.h                                      | 5 +++--
 src/lib/sha1.c                                     | 2 +-
 src/lib/sha1.h                                     | 2 +-
 src/lib/sha2.c                                     | 8 ++++----
 src/lib/sha2.h                                     | 8 ++++----
 src/plugins/pop3-migration/pop3-migration-plugin.c | 6 +++---
 src/plugins/pop3-migration/pop3-migration-plugin.h | 2 +-
 21 files changed, 41 insertions(+), 33 deletions(-)

diff --git a/src/auth/mech-rpa.c b/src/auth/mech-rpa.c
index 743b323f067..98b6f9e4406 100644
--- a/src/auth/mech-rpa.c
+++ b/src/auth/mech-rpa.c
@@ -62,7 +62,7 @@ void *ucs2be_str(pool_t pool, const char *str, size_t *size);
  * Compute client -> server authentication response.
  */
 static void rpa_user_response(struct rpa_auth_request *request,
-			      unsigned char digest[MD5_RESULTLEN])
+			      unsigned char digest[STATIC_ARRAY MD5_RESULTLEN])
 {
 	struct md5_context ctx;
 	unsigned char z[48];
@@ -86,7 +86,7 @@ static void rpa_user_response(struct rpa_auth_request *request,
  * Compute server -> client authentication response.
  */
 static void rpa_server_response(struct rpa_auth_request *request,
-				unsigned char digest[MD5_RESULTLEN])
+				unsigned char digest[STATIC_ARRAY MD5_RESULTLEN])
 {
 	struct md5_context ctx;
 	unsigned char tmp[MD5_RESULTLEN];
diff --git a/src/auth/passdb.c b/src/auth/passdb.c
index ec47e2c3473..a9068d72e1f 100644
--- a/src/auth/passdb.c
+++ b/src/auth/passdb.c
@@ -252,7 +252,7 @@ void passdb_deinit(struct passdb_module *passdb)
 	passdb->iface = passdb_iface_deinit;
 }
 
-void passdbs_generate_md5(unsigned char md5[MD5_RESULTLEN])
+void passdbs_generate_md5(unsigned char md5[STATIC_ARRAY MD5_RESULTLEN])
 {
 	struct md5_context ctx;
 	struct passdb_module *const *passdbs;
diff --git a/src/auth/passdb.h b/src/auth/passdb.h
index 60f05b3b784..57b25391e1e 100644
--- a/src/auth/passdb.h
+++ b/src/auth/passdb.h
@@ -102,7 +102,7 @@ void passdb_deinit(struct passdb_module *passdb);
 void passdb_register_module(struct passdb_module_interface *iface);
 void passdb_unregister_module(struct passdb_module_interface *iface);
 
-void passdbs_generate_md5(unsigned char md5[MD5_RESULTLEN]);
+void passdbs_generate_md5(unsigned char md5[STATIC_ARRAY MD5_RESULTLEN]);
 
 void passdbs_init(void);
 void passdbs_deinit(void);
diff --git a/src/auth/userdb.c b/src/auth/userdb.c
index 5b45ddba728..29e9b2e0668 100644
--- a/src/auth/userdb.c
+++ b/src/auth/userdb.c
@@ -194,7 +194,7 @@ void userdb_deinit(struct userdb_module *userdb)
 	userdb->iface = &userdb_iface_deinit;
 }
 
-void userdbs_generate_md5(unsigned char md5[MD5_RESULTLEN])
+void userdbs_generate_md5(unsigned char md5[STATIC_ARRAY MD5_RESULTLEN])
 {
 	struct md5_context ctx;
 	struct userdb_module *const *userdbs;
diff --git a/src/auth/userdb.h b/src/auth/userdb.h
index 45ba70b3adb..ac527011e87 100644
--- a/src/auth/userdb.h
+++ b/src/auth/userdb.h
@@ -81,7 +81,7 @@ void userdb_deinit(struct userdb_module *userdb);
 void userdb_register_module(struct userdb_module_interface *iface);
 void userdb_unregister_module(struct userdb_module_interface *iface);
 
-void userdbs_generate_md5(unsigned char md5[MD5_RESULTLEN]);
+void userdbs_generate_md5(unsigned char md5[STATIC_ARRAY MD5_RESULTLEN]);
 
 void userdbs_init(void);
 void userdbs_deinit(void);
diff --git a/src/lib-storage/index/mbox/mbox-md5-all.c b/src/lib-storage/index/mbox/mbox-md5-all.c
index e9831ffbf37..c8e744ef803 100644
--- a/src/lib-storage/index/mbox/mbox-md5-all.c
+++ b/src/lib-storage/index/mbox/mbox-md5-all.c
@@ -26,7 +26,7 @@ static void mbox_md5_all_more(struct mbox_md5_context *ctx,
 }
 
 static void mbox_md5_all_finish(struct mbox_md5_context *ctx,
-				unsigned char result[16])
+				unsigned char result[STATIC_ARRAY 16])
 {
 	md5_final(&ctx->hdr_md5_ctx, result);
 	i_free(ctx);
diff --git a/src/lib-storage/index/mbox/mbox-md5-apop3d.c b/src/lib-storage/index/mbox/mbox-md5-apop3d.c
index e980d0b5961..7786a6c195a 100644
--- a/src/lib-storage/index/mbox/mbox-md5-apop3d.c
+++ b/src/lib-storage/index/mbox/mbox-md5-apop3d.c
@@ -106,7 +106,7 @@ static void mbox_md5_apop3d_more(struct mbox_md5_context *ctx,
 }
 
 static void mbox_md5_apop3d_finish(struct mbox_md5_context *ctx,
-				   unsigned char result[16])
+				   unsigned char result[STATIC_ARRAY 16])
 {
 	md5_final(&ctx->hdr_md5_ctx, result);
 	i_free(ctx);
diff --git a/src/lib-storage/index/mbox/mbox-md5.h b/src/lib-storage/index/mbox/mbox-md5.h
index debd2101a32..7584052e64b 100644
--- a/src/lib-storage/index/mbox/mbox-md5.h
+++ b/src/lib-storage/index/mbox/mbox-md5.h
@@ -8,7 +8,7 @@ struct mbox_md5_vfuncs {
 	void (*more)(struct mbox_md5_context *ctx,
 		     struct message_header_line *hdr);
 	void (*finish)(struct mbox_md5_context *ctx,
-		       unsigned char result[16]);
+		       unsigned char result[STATIC_ARRAY 16]);
 };
 
 extern struct mbox_md5_vfuncs mbox_md5_apop3d;
diff --git a/src/lib/guid.c b/src/lib/guid.c
index b0524dbb51f..86d746412e8 100644
--- a/src/lib/guid.c
+++ b/src/lib/guid.c
@@ -36,7 +36,7 @@ const char *guid_generate(void)
 }
 
 void guid_128_host_hash_get(const char *host,
-			    unsigned char hash_r[GUID_128_HOST_HASH_SIZE])
+			    unsigned char hash_r[STATIC_ARRAY GUID_128_HOST_HASH_SIZE])
 {
 	unsigned char full_hash[SHA1_RESULTLEN];
 
diff --git a/src/lib/guid.h b/src/lib/guid.h
index 28fc4a11651..867e3d0cb20 100644
--- a/src/lib/guid.h
+++ b/src/lib/guid.h
@@ -37,6 +37,6 @@ int guid_128_cmp(const guid_128_t guid1, const guid_128_t guid2) ATTR_PURE;
 
 /* Return the hash of host used by guid_128_generate(). */
 void guid_128_host_hash_get(const char *host,
-			    unsigned char hash_r[GUID_128_HOST_HASH_SIZE]);
+			    unsigned char hash_r[STATIC_ARRAY GUID_128_HOST_HASH_SIZE]);
 
 #endif
diff --git a/src/lib/macros.h b/src/lib/macros.h
index f8e4689c193..2faf0c8cf17 100644
--- a/src/lib/macros.h
+++ b/src/lib/macros.h
@@ -226,4 +226,10 @@
 #  define DOVECOT_PREREQ(maj, min) 0
 #endif
 
+#ifdef __cplusplus
+#  define STATIC_ARRAY
+#else
+#  define STATIC_ARRAY static
+#endif
+
 #endif
diff --git a/src/lib/md4.c b/src/lib/md4.c
index fade4d44512..de59b38c81f 100644
--- a/src/lib/md4.c
+++ b/src/lib/md4.c
@@ -208,7 +208,7 @@ void md4_update(struct md4_context *ctx, const void *data, size_t size)
 	memcpy(ctx->buffer, data, size);
 }
 
-void md4_final(struct md4_context *ctx, unsigned char result[MD4_RESULTLEN])
+void md4_final(struct md4_context *ctx, unsigned char result[STATIC_ARRAY MD4_RESULTLEN])
 {
 	/* @UNSAFE */
 	unsigned long used, free;
@@ -261,7 +261,7 @@ void md4_final(struct md4_context *ctx, unsigned char result[MD4_RESULTLEN])
 }
 
 void md4_get_digest(const void *data, size_t size,
-		    unsigned char result[MD4_RESULTLEN])
+		    unsigned char result[STATIC_ARRAY MD4_RESULTLEN])
 {
 	struct md4_context ctx;
 
diff --git a/src/lib/md4.h b/src/lib/md4.h
index 771e08c850e..1530d0d0ff5 100644
--- a/src/lib/md4.h
+++ b/src/lib/md4.h
@@ -22,10 +22,11 @@ struct md4_context {
 
 void md4_init(struct md4_context *ctx);
 void md4_update(struct md4_context *ctx, const void *data, size_t size);
-void md4_final(struct md4_context *ctx, unsigned char result[MD4_RESULTLEN]);
+void md4_final(struct md4_context *ctx,
+		unsigned char result[STATIC_ARRAY MD4_RESULTLEN]);
 
 void md4_get_digest(const void *data, size_t size,
-		    unsigned char result[MD4_RESULTLEN]);
+		    unsigned char result[STATIC_ARRAY MD4_RESULTLEN]);
 
 extern const struct hash_method hash_method_md4;
 
diff --git a/src/lib/md5.c b/src/lib/md5.c
index 6957e054254..16264290c55 100644
--- a/src/lib/md5.c
+++ b/src/lib/md5.c
@@ -220,7 +220,7 @@ void md5_update(struct md5_context *ctx, const void *data, size_t size)
 	memcpy(ctx->buffer, data, size);
 }
 
-void md5_final(struct md5_context *ctx, unsigned char result[MD5_RESULTLEN])
+void md5_final(struct md5_context *ctx, unsigned char result[STATIC_ARRAY MD5_RESULTLEN])
 {
 	/* @UNSAFE */
 	unsigned long used, free;
@@ -273,7 +273,7 @@ void md5_final(struct md5_context *ctx, unsigned char result[MD5_RESULTLEN])
 }
 
 void md5_get_digest(const void *data, size_t size,
-		    unsigned char result[MD5_RESULTLEN])
+		    unsigned char result[STATIC_ARRAY MD5_RESULTLEN])
 {
 	struct md5_context ctx;
 
diff --git a/src/lib/md5.h b/src/lib/md5.h
index 3da87fba375..682a6c5fe94 100644
--- a/src/lib/md5.h
+++ b/src/lib/md5.h
@@ -22,10 +22,11 @@ struct md5_context {
 
 void md5_init(struct md5_context *ctx);
 void md5_update(struct md5_context *ctx, const void *data, size_t size);
-void md5_final(struct md5_context *ctx, unsigned char result[MD5_RESULTLEN]);
+void md5_final(struct md5_context *ctx,
+	       unsigned char result[STATIC_ARRAY MD5_RESULTLEN]);
 
 void md5_get_digest(const void *data, size_t size,
-		    unsigned char result[MD5_RESULTLEN]);
+		    unsigned char result[STATIC_ARRAY MD5_RESULTLEN]);
 
 extern const struct hash_method hash_method_md5;
 
diff --git a/src/lib/sha1.c b/src/lib/sha1.c
index 947f3f7d8fb..edb1735e7fb 100644
--- a/src/lib/sha1.c
+++ b/src/lib/sha1.c
@@ -253,7 +253,7 @@ sha1_result(struct sha1_ctxt *ctxt, void *digest0)
 }
 
 void sha1_get_digest(const void *data, size_t size,
-		     unsigned char result[SHA1_RESULTLEN])
+		     unsigned char result[STATIC_ARRAY SHA1_RESULTLEN])
 {
 	struct sha1_ctxt ctx;
 
diff --git a/src/lib/sha1.h b/src/lib/sha1.h
index 539f3c50c53..73b3110582c 100644
--- a/src/lib/sha1.h
+++ b/src/lib/sha1.h
@@ -77,7 +77,7 @@ typedef struct sha1_ctxt SHA1_CTX;
 #define	SHA1_RESULTLEN	(160/8)
 
 extern void sha1_get_digest(const void *, size_t,
-	unsigned char [SHA1_RESULTLEN]);
+	unsigned char [STATIC_ARRAY SHA1_RESULTLEN]);
 
 extern const struct hash_method hash_method_sha1;
 
diff --git a/src/lib/sha2.c b/src/lib/sha2.c
index 9ea500d6f09..c3aa91dee54 100644
--- a/src/lib/sha2.c
+++ b/src/lib/sha2.c
@@ -265,7 +265,7 @@ void sha256_loop(struct sha256_ctx *ctx, const void *data,
 }
 
 void sha256_result(struct sha256_ctx *ctx,
-		   unsigned char digest[SHA256_RESULTLEN])
+		   unsigned char digest[STATIC_ARRAY SHA256_RESULTLEN])
 {
 	size_t block_nb;
 	size_t pm_len;
@@ -290,7 +290,7 @@ void sha256_result(struct sha256_ctx *ctx,
 }
 
 void sha256_get_digest(const void *data, size_t size,
-		       unsigned char digest[SHA256_RESULTLEN])
+		       unsigned char digest[STATIC_ARRAY SHA256_RESULTLEN])
 {
 	struct sha256_ctx ctx;
 
@@ -390,7 +390,7 @@ void sha512_loop(struct sha512_ctx *ctx, const void *data,
 }
 
 void sha512_result(struct sha512_ctx *ctx,
-		   unsigned char digest[SHA512_RESULTLEN])
+		   unsigned char digest[STATIC_ARRAY SHA512_RESULTLEN])
 {
 	unsigned int block_nb;
 	unsigned int pm_len;
@@ -415,7 +415,7 @@ void sha512_result(struct sha512_ctx *ctx,
 }
 
 void sha512_get_digest(const void *data, size_t size,
-		       unsigned char digest[SHA512_RESULTLEN])
+		       unsigned char digest[STATIC_ARRAY SHA512_RESULTLEN])
 {
 	struct sha512_ctx ctx;
 
diff --git a/src/lib/sha2.h b/src/lib/sha2.h
index 3ee937441d6..b04f93ae5a0 100644
--- a/src/lib/sha2.h
+++ b/src/lib/sha2.h
@@ -59,18 +59,18 @@ struct sha512_ctx {
 void sha256_init(struct sha256_ctx *ctx);
 void sha256_loop(struct sha256_ctx *ctx, const void *data, size_t len);
 void sha256_result(struct sha256_ctx *ctx,
-		   unsigned char digest[SHA256_RESULTLEN]);
+		   unsigned char digest[STATIC_ARRAY SHA256_RESULTLEN]);
 
 void sha256_get_digest(const void *data, size_t size,
-		       unsigned char digest[SHA256_RESULTLEN]);
+		       unsigned char digest[STATIC_ARRAY SHA256_RESULTLEN]);
 
 void sha512_init(struct sha512_ctx *ctx);
 void sha512_loop(struct sha512_ctx *ctx, const void *data, size_t len);
 void sha512_result(struct sha512_ctx *ctx,
-		   unsigned char digest[SHA512_RESULTLEN]);
+		   unsigned char digest[STATIC_ARRAY SHA512_RESULTLEN]);
 
 void sha512_get_digest(const void *data, size_t size,
-		       unsigned char digest[SHA512_RESULTLEN]);
+		       unsigned char digest[STATIC_ARRAY SHA512_RESULTLEN]);
 
 extern const struct hash_method hash_method_sha256;
 extern const struct hash_method hash_method_sha512;
diff --git a/src/plugins/pop3-migration/pop3-migration-plugin.c b/src/plugins/pop3-migration/pop3-migration-plugin.c
index 8a2795a4f74..5e0e2b33a39 100644
--- a/src/plugins/pop3-migration/pop3-migration-plugin.c
+++ b/src/plugins/pop3-migration/pop3-migration-plugin.c
@@ -177,7 +177,7 @@ pop3_header_filter_callback(struct header_filter_istream *input ATTR_UNUSED,
 
 int pop3_migration_get_hdr_sha1(uint32_t mail_seq, struct istream *input,
 				uoff_t hdr_size,
-				unsigned char sha1_r[SHA1_RESULTLEN],
+				unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN],
 				bool *have_eoh_r)
 {
 	struct istream *input2;
@@ -231,7 +231,7 @@ static unsigned int get_cache_idx(struct mail *mail)
 }
 
 static int
-get_hdr_sha1(struct mail *mail, unsigned char sha1_r[SHA1_RESULTLEN])
+get_hdr_sha1(struct mail *mail, unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN])
 {
 	struct istream *input;
 	struct message_size hdr_size;
@@ -290,7 +290,7 @@ get_hdr_sha1(struct mail *mail, unsigned char sha1_r[SHA1_RESULTLEN])
 
 static bool
 get_cached_hdr_sha1(struct mail *mail, buffer_t *cache_buf,
-		    unsigned char sha1_r[SHA1_RESULTLEN])
+		    unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN])
 {
 	struct index_mail *imail = (struct index_mail *)mail;
 
diff --git a/src/plugins/pop3-migration/pop3-migration-plugin.h b/src/plugins/pop3-migration/pop3-migration-plugin.h
index 510eca00a89..398ffb9c4e2 100644
--- a/src/plugins/pop3-migration/pop3-migration-plugin.h
+++ b/src/plugins/pop3-migration/pop3-migration-plugin.h
@@ -8,7 +8,7 @@ void pop3_migration_plugin_deinit(void);
 
 int pop3_migration_get_hdr_sha1(uint32_t mail_seq, struct istream *input,
 				uoff_t hdr_size,
-				unsigned char sha1_r[SHA1_RESULTLEN],
+				unsigned char sha1_r[STATIC_ARRAY SHA1_RESULTLEN],
 				bool *have_eoh_r);
 
 #endif

From c36510421861bda8fbdfb49d0dcd55f89bee0cb7 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 14 Jun 2016 22:32:36 +0300
Subject: [PATCH 301/450] lib-fs: Added write_bytes to statistics

---
 src/lib-fs/fs-api.c | 2 ++
 src/lib-fs/fs-api.h | 3 +++
 2 files changed, 5 insertions(+)

diff --git a/src/lib-fs/fs-api.c b/src/lib-fs/fs-api.c
index dc892d2014f..7bd981eb3b0 100644
--- a/src/lib-fs/fs-api.c
+++ b/src/lib-fs/fs-api.c
@@ -608,6 +608,7 @@ int fs_write(struct fs_file *file, const void *data, size_t size)
 		} T_END;
 		if (!(ret < 0 && errno == EAGAIN)) {
 			file->fs->stats.write_count++;
+			file->fs->stats.write_bytes += size;
 			fs_file_timing_end(file, FS_OP_WRITE);
 		}
 		return ret;
@@ -674,6 +675,7 @@ int fs_write_stream_finish(struct fs_file *file, struct ostream **output)
 				     o_stream_get_error(file->output));
 			success = FALSE;
 		}
+		file->fs->stats.write_bytes += file->output->offset;
 	}
 	return fs_write_stream_finish_int(file, success);
 }
diff --git a/src/lib-fs/fs-api.h b/src/lib-fs/fs-api.h
index 2f8da389cc8..21979d4b14c 100644
--- a/src/lib-fs/fs-api.h
+++ b/src/lib-fs/fs-api.h
@@ -175,6 +175,9 @@ struct fs_stats {
 	/* Number of fs_iter_init() calls. */
 	unsigned int iter_count;
 
+	/* Number of bytes written by fs_write*() calls. */
+	uint64_t write_bytes;
+
 	/* Cumulative sum of usecs spent on calls - set only if
 	   fs_settings.enable_timing=TRUE */
 	struct timing *timings[FS_OP_COUNT];

From d737445a4a787171cdca13ce3de366369bf493fb Mon Sep 17 00:00:00 2001
From: Teemu Huovila <teemu.huovila@dovecot.fi>
Date: Tue, 14 Jun 2016 22:42:09 +0300
Subject: [PATCH 302/450] lib-dcrypt: Fix error_r pointer verification.

---
 src/lib-dcrypt/dcrypt.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt.c b/src/lib-dcrypt/dcrypt.c
index e8b2d00bfe0..4312848cdb2 100644
--- a/src/lib-dcrypt/dcrypt.c
+++ b/src/lib-dcrypt/dcrypt.c
@@ -22,12 +22,12 @@ bool dcrypt_initialize(const char *backend, const char **error_r)
 	mod_set.require_init_funcs = 1;
 	dcrypt_module = module_dir_load(DCRYPT_MODULE_DIR, implementation, &mod_set);
 	if (dcrypt_module == NULL) {
-		if (*error_r != NULL)
+		if (error_r != NULL)
 			*error_r = "No such module";
 		return FALSE;
 	}
 	if (dcrypt_module->init == NULL) {
-		if (*error_r != NULL)
+		if (error_r != NULL)
 			*error_r = "Module missing init/deinit";
 		return FALSE;
 	}

From cf757d601e94987bca6c26a353221abc88e37762 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Wed, 15 Jun 2016 01:53:44 +0200
Subject: [PATCH 303/450] time-util: Fixed usec comparison in
 timeval_cmp_margin().

---
 src/lib/time-util.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/lib/time-util.c b/src/lib/time-util.c
index 9bfad68558c..3ab1a86821a 100644
--- a/src/lib/time-util.c
+++ b/src/lib/time-util.c
@@ -28,10 +28,10 @@ int timeval_cmp_margin(const struct timeval *tv1, const struct timeval *tv2,
 	if (tv1->tv_sec > tv2->tv_sec)
 		return 1;
 
-	if (tv1->tv_usec - tv2->tv_usec < (int)usec_margin)
-		return -1;
-	if (tv1->tv_usec - tv2->tv_usec > (int)usec_margin)
+	if ((tv2->tv_usec - tv1->tv_usec) > (int)usec_margin)
 		return -1;
+	if ((tv1->tv_usec - tv2->tv_usec) > (int)usec_margin)
+		return 1;
 	return 0;
 }
 

From 4889576ced68bcccac13c4d8e7decf700bc974c1 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Wed, 15 Jun 2016 01:54:25 +0200
Subject: [PATCH 304/450] lib-http: client: Added more debug logging about the
 submission of delayed requests.

---
 src/lib-http/http-client-queue.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/lib-http/http-client-queue.c b/src/lib-http/http-client-queue.c
index 42f24e11600..153c80074ec 100644
--- a/src/lib-http/http-client-queue.c
+++ b/src/lib-http/http-client-queue.c
@@ -743,6 +743,12 @@ void http_client_queue_submit_request(struct http_client_queue *queue,
 
 		if (timeval_cmp_margin(&req->release_time,
 			&ioloop_timeval, TIMEOUT_CMP_MARGIN_USECS) > 0) {
+			http_client_queue_debug(queue,
+				"Delayed request %s%s submitted (time remaining: %d msecs)",
+				http_client_request_label(req),
+				(req->urgent ? " (urgent)" : ""),
+				timeval_diff_msecs(&req->release_time, &ioloop_timeval));
+
 			(void)array_bsearch_insert_pos(&queue->delayed_requests,
 					&req, http_client_queue_delayed_cmp, &insert_idx);
 			array_insert(&queue->delayed_requests, insert_idx, &req, 1);

From 97559e0a29df6db753f359d135fe2f53b1574e89 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 19 Jun 2016 18:38:39 +0300
Subject: [PATCH 305/450] doveadm dump: Updated obox's oid output.

---
 src/doveadm/doveadm-dump-index.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/doveadm/doveadm-dump-index.c b/src/doveadm/doveadm-dump-index.c
index 21509cefbaa..3e048444c4a 100644
--- a/src/doveadm/doveadm-dump-index.c
+++ b/src/doveadm/doveadm-dump-index.c
@@ -621,7 +621,7 @@ static void dump_record(struct mail_index_view *view, unsigned int seq)
 		} else if (strcmp(ext[i].name, "obox") == 0) {
 			const struct obox_mail_index_record *orec = data;
 			printf("                   : guid = %s\n", guid_128_to_string(orec->guid));
-			printf("                   : oid  = %s\n", guid_128_to_string(orec->oid));
+			printf("                   : oid  = %s\n", binary_to_hex(orec->oid, ext[i].record_size - sizeof(orec->guid)));
 		} else if (strcmp(ext[i].name, "mobox") == 0) {
 			const struct mobox_mail_index_record *orec = data;
 			printf("                   : map_uid   = %u\n", orec->map_uid);

From 6fd4b93aa0b1dcb306bebed51e2ae803defbf302 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 19 Jun 2016 18:44:55 +0300
Subject: [PATCH 306/450] lib-dcrypt: Fixed memory leak in test-crypto unit
 test

---
 src/lib-dcrypt/test-crypto.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/lib-dcrypt/test-crypto.c b/src/lib-dcrypt/test-crypto.c
index d8432869e94..4e2e6885a6f 100644
--- a/src/lib-dcrypt/test-crypto.c
+++ b/src/lib-dcrypt/test-crypto.c
@@ -301,6 +301,8 @@ void test_load_v2_public_key(void)
 	test_assert(dcrypt_key_store_public(pub, DCRYPT_FORMAT_DOVECOT, tmp, &error));
 
 	test_assert(strcmp(key, str_c(tmp))==0);
+	buffer_free(&tmp);
+	dcrypt_key_free_public(&pub);
 
 	test_end();
 }

From c70202530e329177d24a4bd2073600d67c599afb Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 19 Jun 2016 20:23:27 +0300
Subject: [PATCH 307/450] mailbox-alias: Fixed renaming mailboxes when the
 plugin is loaded.

---
 src/plugins/mailbox-alias/mailbox-alias-plugin.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plugins/mailbox-alias/mailbox-alias-plugin.c b/src/plugins/mailbox-alias/mailbox-alias-plugin.c
index d7ee7f9deb5..2148b0ac684 100644
--- a/src/plugins/mailbox-alias/mailbox-alias-plugin.c
+++ b/src/plugins/mailbox-alias/mailbox-alias-plugin.c
@@ -256,7 +256,7 @@ static int mailbox_alias_rename(struct mailbox *src, struct mailbox *dest)
 	}
 	if ((ret = mailbox_is_alias_symlink(dest)) < 0)
 		return -1;
-	else {
+	else if (ret > 0) {
 		mail_storage_set_error(src->storage, MAIL_ERROR_NOTPOSSIBLE,
 				       "Can't rename to mailbox alias");
 		return -1;

From a12763b80dca2ce7ccfbcfdf133caa728e9f26e3 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 19 Jun 2016 23:18:43 +0300
Subject: [PATCH 308/450] lib-dcrypt: Fixed running unit tests in build
 directory.

---
 src/lib-dcrypt/Makefile.am   | 4 ++--
 src/lib-dcrypt/test-stream.c | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/lib-dcrypt/Makefile.am b/src/lib-dcrypt/Makefile.am
index ea75e0c1f92..c7618f0dc5c 100644
--- a/src/lib-dcrypt/Makefile.am
+++ b/src/lib-dcrypt/Makefile.am
@@ -52,9 +52,9 @@ LIBDOVECOT_TEST = \
 	$(MODULE_LIBS)
 
 test_crypto_LDADD = $(LIBDOVECOT_TEST)
-test_crypto_CFLAGS = $(AM_CPPFLAGS) -DDCRYPT_MODULE_DIR=\"$(top_srcdir)/src/lib-dcrypt/.libs\"
+test_crypto_CFLAGS = $(AM_CPPFLAGS) -DDCRYPT_MODULE_DIR=\".libs\" -DDCRYPT_SRC_DIR=\"$(top_srcdir)/src/lib-dcrypt\"
 test_crypto_SOURCES = $(libdcrypt_la_SOURCES) test-crypto.c
 
 test_stream_LDADD = $(LIBDOVECOT_TEST)
-test_stream_CFLAGS = $(AM_CPPFLAGS) -DDCRYPT_MODULE_DIR=\"$(top_srcdir)/src/lib-dcrypt/.libs\"
+test_stream_CFLAGS = $(AM_CPPFLAGS) -DDCRYPT_MODULE_DIR=\".libs\" -DDCRYPT_SRC_DIR=\"$(top_srcdir)/src/lib-dcrypt\"
 test_stream_SOURCES = $(libdcrypt_la_SOURCES) test-stream.c
diff --git a/src/lib-dcrypt/test-stream.c b/src/lib-dcrypt/test-stream.c
index 1058b30b7ba..959f4cab034 100644
--- a/src/lib-dcrypt/test-stream.c
+++ b/src/lib-dcrypt/test-stream.c
@@ -57,7 +57,7 @@ void test_static_v1_input(void)
 
 	test_begin("test_static_v1_input");
 
-	struct istream *is_1 = i_stream_create_file("sample-v1.asc", IO_BLOCK_SIZE);
+	struct istream *is_1 = i_stream_create_file(DCRYPT_SRC_DIR"/sample-v1.asc", IO_BLOCK_SIZE);
 	struct istream *is_2 = i_stream_create_base64_decoder(is_1);
 	i_stream_unref(&is_1);
 	struct istream *is_3 = i_stream_create_decrypt(is_2, test_v1_kp.priv);
@@ -91,7 +91,7 @@ void test_static_v2_input(void)
 	unsigned char hash_dgst[hash->digest_size];
 	hash->init(hash_ctx);
 
-	struct istream *is_1 = i_stream_create_file("sample-v2.asc", IO_BLOCK_SIZE);
+	struct istream *is_1 = i_stream_create_file(DCRYPT_SRC_DIR"/sample-v2.asc", IO_BLOCK_SIZE);
 	struct istream *is_2 = i_stream_create_base64_decoder(is_1);
 	i_stream_unref(&is_1);
 	struct istream *is_3 = i_stream_create_decrypt(is_2, test_v2_kp.priv);

From c5bb8ef0a2cb42c5cf977dc24963ae95b42a3b89 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 01:47:59 +0300
Subject: [PATCH 309/450] lib-index: Don't break indexes on syscall failures
 during index refreshing.

Especially mmap() failures due to out of memory could have triggered this.
We treated the open as successful, which meant that an empty index was
opened.
---
 src/lib-index/mail-index-sync-update.c | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/lib-index/mail-index-sync-update.c b/src/lib-index/mail-index-sync-update.c
index 05d33bc2bbc..76cb8514c73 100644
--- a/src/lib-index/mail-index-sync-update.c
+++ b/src/lib-index/mail-index-sync-update.c
@@ -955,6 +955,11 @@ int mail_index_sync_map(struct mail_index_map **_map,
 					    &reset, &reason);
 	if (ret <= 0) {
 		mail_index_view_close(&view);
+		if (force && ret < 0) {
+			/* if we failed because of a syscall error, make sure
+			   we return a failure. */
+			return -1;
+		}
 		if (force && ret == 0) {
 			/* the seq/offset is probably broken */
 			mail_index_set_error(index, "Index %s: Lost log for "

From 81b96aa2b01d2f5035ee330365480c09084142dd Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 02:25:47 +0300
Subject: [PATCH 310/450] lib-dcrypt: Use a more supported EC curve in unit
 test

Fixes running the test on CentOS 6.
---
 src/lib-dcrypt/test-crypto.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-dcrypt/test-crypto.c b/src/lib-dcrypt/test-crypto.c
index 4e2e6885a6f..a7199dedc4b 100644
--- a/src/lib-dcrypt/test-crypto.c
+++ b/src/lib-dcrypt/test-crypto.c
@@ -292,7 +292,7 @@ void test_load_v2_public_key(void)
 	const char *error;
 
 	test_begin("test_load_v2_public_key");
-	const char *key = "2\t305e301006072a8648ce3d020106052b81040026034a000303a9288126a4ef239199d7ebe784d0b81b545df40e1feac5980965914524005fd11d18cf71cfd875a037172275dda474bcf6a96fd4824c9019b108e5258c0548ee70c6ce1d67ca5d";
+	const char *key = "2\t3058301006072a8648ce3d020106052b810400230344000301c50954e734dd8b410a607764a7057065a45510da52f2c6e28e0cb353b9c389fa8cb786943ae991fce9befed78fb162fbbc615415f06af06c8cc80c37f4e94ff6c7";
 
 	test_assert(dcrypt_key_load_public(&pub, DCRYPT_FORMAT_DOVECOT, key, &error));
 

From 7ebd546ed6e9c8185e8d5c7eb24b637ea495ae06 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 6 May 2016 17:24:42 +0300
Subject: [PATCH 311/450] lib: Added t_str_tabunescape()

---
 src/lib/strescape.c      | 8 ++++++++
 src/lib/strescape.h      | 1 +
 src/lib/test-strescape.c | 2 ++
 3 files changed, 11 insertions(+)

diff --git a/src/lib/strescape.c b/src/lib/strescape.c
index 065a50910b1..75641c21e36 100644
--- a/src/lib/strescape.c
+++ b/src/lib/strescape.c
@@ -226,6 +226,14 @@ char *str_tabunescape(char *str)
 	return start;
 }
 
+const char *t_str_tabunescape(const char *str)
+{
+	if (strchr(str, '\001') == NULL)
+		return str;
+	else
+		return str_tabunescape(t_strdup_noconst(str));
+}
+
 char **p_strsplit_tabescaped(pool_t pool, const char *str)
 {
 	char **args;
diff --git a/src/lib/strescape.h b/src/lib/strescape.h
index 9569b94fd45..550e3adfac3 100644
--- a/src/lib/strescape.h
+++ b/src/lib/strescape.h
@@ -23,6 +23,7 @@ const char *str_tabescape(const char *str);
 void str_append_tabescaped(string_t *dest, const char *src);
 void str_append_tabunescaped(string_t *dest, const void *src, size_t src_size);
 char *str_tabunescape(char *str);
+const char *t_str_tabunescape(const char *str);
 
 char **p_strsplit_tabescaped(pool_t pool, const char *str);
 const char *const *t_strsplit_tabescaped(const char *str);
diff --git a/src/lib/test-strescape.c b/src/lib/test-strescape.c
index c4c5c3905af..4272cae3b26 100644
--- a/src/lib/test-strescape.c
+++ b/src/lib/test-strescape.c
@@ -70,6 +70,8 @@ void test_strescape(void)
 
 	test_begin("str_tabescape");
 	for (i = 0; i < N_ELEMENTS(tabesc); i++) {
+		test_assert(strcmp(t_str_tabunescape(tabesc[i].output),
+				   tabesc[i].input) == 0);
 		test_assert(strcmp(str_tabunescape(t_strdup_noconst(tabesc[i].output)),
 				   tabesc[i].input) == 0);
 		test_assert(strcmp(str_tabescape(tabesc[i].input),

From d05c72b4547763ed0a7f859e6690d12aa2ec1ba8 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 2 Jun 2016 00:57:17 +0300
Subject: [PATCH 312/450] lib-dict: dict-client rewrite to support async
 operations

---
 src/lib-dict/dict-client.c | 1233 +++++++++++++++++++-----------------
 1 file changed, 657 insertions(+), 576 deletions(-)

diff --git a/src/lib-dict/dict-client.c b/src/lib-dict/dict-client.c
index b4ceb038f2d..e1716b8f9cf 100644
--- a/src/lib-dict/dict-client.c
+++ b/src/lib-dict/dict-client.c
@@ -1,10 +1,12 @@
 /* Copyright (c) 2005-2016 Dovecot authors, see the included COPYING file */
 
 #include "lib.h"
+#include "array.h"
 #include "llist.h"
 #include "str.h"
-#include "net.h"
-#include "istream.h"
+#include "strescape.h"
+#include "time-util.h"
+#include "connection.h"
 #include "ostream.h"
 #include "eacces-error.h"
 #include "dict-private.h"
@@ -21,390 +23,291 @@
 #define DICT_CLIENT_DEFAULT_TIMEOUT_MSECS 0
 
 /* Abort dict lookup after this many seconds. */
-#define DICT_CLIENT_READ_TIMEOUT_SECS 30
-/* Log a warning if dict lookup takes longer than this many seconds. */
-#define DICT_CLIENT_READ_WARN_TIMEOUT_SECS 5
+#define DICT_CLIENT_REQUEST_TIMEOUT_MSECS 30000
+/* Log a warning if dict lookup takes longer than this many milliseconds. */
+#define DICT_CLIENT_REQUEST_WARN_TIMEOUT_MSECS 5000
+
+struct client_dict_cmd {
+	int refcount;
+	struct client_dict *dict;
+	struct timeval start_time;
+	char *query;
+
+	bool retry_errors;
+	bool no_replies;
+	bool unfinished;
+
+	void (*callback)(struct client_dict_cmd *cmd,
+			 const char *line, const char *error);
+        struct client_dict_iterate_context *iter;
+
+	struct {
+		dict_lookup_callback_t *lookup;
+		dict_transaction_commit_callback_t *commit;
+		void *context;
+	} api_callback;
+};
+
+struct dict_connection {
+	struct connection conn;
+	struct client_dict *dict;
+};
 
 struct client_dict {
 	struct dict dict;
+	struct dict_connection conn;
 
-	pool_t pool;
-	int fd;
-	const char *uri;
-	const char *username;
-	const char *path;
+	char *uri, *username;
 	enum dict_data_type value_type;
 
 	time_t last_failed_connect;
-	struct istream *input;
-	struct ostream *output;
-	struct io *io;
+	struct ioloop *ioloop, *prev_ioloop;
+	struct timeout *to_requests;
 	struct timeout *to_idle;
 	unsigned int idle_msecs;
+	struct timeval last_input;
 
+	ARRAY(struct client_dict_cmd *) cmds;
 	struct client_dict_transaction_context *transactions;
 
-	unsigned int connect_counter;
 	unsigned int transaction_id_counter;
-	unsigned int async_commits;
-	unsigned int iter_replies_skip;
+};
 
-	unsigned int in_iteration:1;
-	unsigned int handshaked:1;
+struct client_dict_iter_result {
+	const char *key, *value;
 };
 
 struct client_dict_iterate_context {
 	struct dict_iterate_context ctx;
+	char *error;
+
+	pool_t results_pool;
+	ARRAY(struct client_dict_iter_result) results;
+	unsigned int result_idx;
 
-	pool_t pool;
-	bool failed;
+	bool async;
 	bool finished;
+	bool deinit;
 };
 
 struct client_dict_transaction_context {
 	struct dict_transaction_context ctx;
 	struct client_dict_transaction_context *prev, *next;
 
-	/* for async commits */
-	dict_transaction_commit_callback_t *callback;
-	void *context;
+	char *error;
 
 	unsigned int id;
-	unsigned int connect_counter;
 
-	unsigned int failed:1;
-	unsigned int sent_begin:1;
-	unsigned int async:1;
-	unsigned int committed:1;
+	bool sent_begin:1;
 };
 
-static int client_dict_connect(struct client_dict *dict);
-static void client_dict_disconnect(struct client_dict *dict);
-
-const char *dict_client_escape(const char *src)
-{
-	const char *p;
-	string_t *dest;
-
-	/* first do a quick lookup to see if there's anything to escape.
-	   probably not. */
-	for (p = src; *p != '\0'; p++) {
-		if (*p == '\t' || *p == '\n' || *p == '\001')
-			break;
-	}
-
-	if (*p == '\0')
-		return src;
-
-	dest = t_str_new(256);
-	str_append_n(dest, src, p - src);
-
-	for (; *p != '\0'; p++) {
-		switch (*p) {
-		case '\t':
-			str_append_c(dest, '\001');
-			str_append_c(dest, 't');
-			break;
-		case '\n':
-			str_append_c(dest, '\001');
-			str_append_c(dest, 'n');
-			break;
-		case '\001':
-			str_append_c(dest, '\001');
-			str_append_c(dest, '1');
-			break;
-		default:
-			str_append_c(dest, *p);
-			break;
-		}
-	}
-	return str_c(dest);
-}
-
-const char *dict_client_unescape(const char *src)
-{
-	const char *p;
-	string_t *dest;
-
-	/* first do a quick lookup to see if there's anything to unescape.
-	   probably not. */
-	for (p = src; *p != '\0'; p++) {
-		if (*p == '\001')
-			break;
-	}
-
-	if (*p == '\0')
-		return src;
-
-	dest = t_str_new(256);
-	str_append_n(dest, src, p - src);
-	for (; *p != '\0'; p++) {
-		if (*p != '\001')
-			str_append_c(dest, *p);
-		else if (p[1] != '\0') {
-			p++;
-			switch (*p) {
-			case '1':
-				str_append_c(dest, '\001');
-				break;
-			case 't':
-				str_append_c(dest, '\t');
-				break;
-			case 'n':
-				str_append_c(dest, '\n');
-				break;
-			}
-		}
-	}
-	return str_c(dest);
-}
-
-static int client_dict_send_query(struct client_dict *dict, const char *query)
-{
-	if (dict->output == NULL) {
-		/* not connected currently */
-		if (client_dict_connect(dict) < 0)
-			return -1;
-	}
+static struct connection_list *dict_connections;
 
-	if (o_stream_send_str(dict->output, query) < 0 ||
-	    o_stream_flush(dict->output) < 0) {
-		/* Send failed */
-		if (!dict->handshaked) {
-			/* we're trying to send hello, don't try to reconnect */
-			return -1;
-		}
+static int client_dict_connect(struct client_dict *dict, const char **error_r);
+static void client_dict_disconnect(struct client_dict *dict, const char *reason);
 
-		/* Reconnect and try again. */
-		client_dict_disconnect(dict);
-		if (client_dict_connect(dict) < 0)
-			return -1;
-
-		if (o_stream_send_str(dict->output, query) < 0 ||
-		    o_stream_flush(dict->output) < 0) {
-			i_error("write(%s) failed: %m", dict->path);
-			client_dict_disconnect(dict);
-			return -1;
-		}
-	}
-	return 0;
+static struct client_dict_cmd *
+client_dict_cmd_init(struct client_dict *dict, const char *query)
+{
+	struct client_dict_cmd *cmd;
+
+	cmd = i_new(struct client_dict_cmd, 1);
+	cmd->refcount = 1;
+	cmd->dict = dict;
+	cmd->query = i_strdup(query);
+	cmd->start_time = ioloop_timeval;
+	return cmd;
 }
 
-static int
-client_dict_transaction_send_begin(struct client_dict_transaction_context *ctx)
+static void client_dict_cmd_ref(struct client_dict_cmd *cmd)
 {
-	struct client_dict *dict = (struct client_dict *)ctx->ctx.dict;
-
-	if (ctx->failed)
-		return -1;
-
-	T_BEGIN {
-		const char *query;
-
-		query = t_strdup_printf("%c%u\n", DICT_PROTOCOL_CMD_BEGIN,
-					ctx->id);
-		if (client_dict_send_query(dict, query) < 0)
-			ctx->failed = TRUE;
-		else
-			ctx->connect_counter = dict->connect_counter;
-	} T_END;
-
-	return ctx->failed ? -1 : 0;
+	i_assert(cmd->refcount > 0);
+	cmd->refcount++;
 }
 
-static int ATTR_NOWARN_UNUSED_RESULT
-client_dict_send_transaction_query(struct client_dict_transaction_context *ctx,
-				   const char *query)
+static bool client_dict_cmd_unref(struct client_dict_cmd *cmd)
 {
-	struct client_dict *dict = (struct client_dict *)ctx->ctx.dict;
+	i_assert(cmd->refcount > 0);
+	if (--cmd->refcount > 0)
+		return TRUE;
 
-	if (!ctx->sent_begin) {
-		if (client_dict_transaction_send_begin(ctx) < 0)
-			return -1;
-		ctx->sent_begin = TRUE;
-	}
-
-	if (ctx->connect_counter != dict->connect_counter || ctx->failed)
-		return -1;
+	i_free(cmd->query);
+	i_free(cmd);
+	return FALSE;
+}
 
-	if (dict->output == NULL) {
-		/* not connected, this'll fail */
-		return -1;
+static void dict_pre_api_callback(struct client_dict *dict)
+{
+	if (dict->prev_ioloop != NULL) {
+		/* Don't let callback see that we've created our
+		   internal ioloop in case it wants to add some ios
+		   or timeouts. */
+		current_ioloop = dict->prev_ioloop;
 	}
+}
 
-	if (o_stream_send_str(dict->output, query) < 0 ||
-	    o_stream_flush(dict->output) < 0) {
-		/* Send failed. Our transactions have died, so don't even try
-		   to re-send the command */
-		ctx->failed = TRUE;
-		client_dict_disconnect(dict);
-		return -1;
+static void dict_post_api_callback(struct client_dict *dict)
+{
+	if (dict->prev_ioloop != NULL) {
+		current_ioloop = dict->ioloop;
+		/* stop client_dict_wait() */
+		io_loop_stop(dict->ioloop);
 	}
-	return 0;
 }
 
-static struct client_dict_transaction_context *
-client_dict_transaction_find(struct client_dict *dict, unsigned int id)
+static bool
+dict_cmd_callback_line(struct client_dict_cmd *cmd, const char *line)
 {
-	struct client_dict_transaction_context *ctx;
-
-	for (ctx = dict->transactions; ctx != NULL; ctx = ctx->next) {
-		if (ctx->id == id)
-			return ctx;
-	}
-	return NULL;
+	cmd->unfinished = FALSE;
+	cmd->callback(cmd, line, NULL);
+	return !cmd->unfinished;
 }
 
 static void
-client_dict_finish_transaction(struct client_dict *dict,
-			       unsigned int id, int ret)
+dict_cmd_callback_error(struct client_dict_cmd *cmd, const char *error)
 {
-	struct client_dict_transaction_context *ctx;
-
-	ctx = client_dict_transaction_find(dict, id);
-	if (ctx == NULL) {
-		i_error("dict-client: Unknown transaction id %u", id);
-		return;
-	}
-	ctx->failed = TRUE;
-	if (!ctx->committed)
-		return;
+	cmd->unfinished = FALSE;
+	if (cmd->callback != NULL)
+		cmd->callback(cmd, NULL, error);
+	i_assert(!cmd->unfinished);
+}
 
-	/* the callback may call the dict code again, so remove this
-	   transaction before calling it */
-	i_assert(dict->async_commits > 0);
-	if (--dict->async_commits == 0) {
-		if (dict->io != NULL)
-			io_remove(&dict->io);
-	}
-	DLLIST_REMOVE(&dict->transactions, ctx);
+static void client_dict_input_timeout(struct client_dict *dict)
+{
+	int diff = timeval_diff_msecs(&ioloop_timeval, &dict->last_input);
 
-	if (ctx->callback != NULL)
-		ctx->callback(ret, ctx->context);
-	i_free(ctx);
+	client_dict_disconnect(dict, t_strdup_printf(
+		"Timeout: No input from dict for %u.%03u secs",
+		diff/1000, diff%1000));
 }
 
-static ssize_t client_dict_read_timeout(struct client_dict *dict)
+static int
+client_dict_cmd_query_send(struct client_dict *dict, const char *query)
 {
-	time_t now, timeout;
-	unsigned int diff;
+	struct const_iovec iov[2];
 	ssize_t ret;
 
-	now = time(NULL);
-	timeout = now + DICT_CLIENT_READ_TIMEOUT_SECS;
-
-	do {
-		alarm(timeout - now);
-		ret = i_stream_read(dict->input);
-		alarm(0);
-		if (ret != 0)
-			break;
-
-		/* interrupted most likely because of timeout,
-		   but check anyway. */
-		now = time(NULL);
-	} while (now < timeout);
-
-	if (ret > 0) {
-		diff = time(NULL) - now;
-		if (diff >= DICT_CLIENT_READ_WARN_TIMEOUT_SECS) {
-			i_warning("read(%s): dict lookup took %u seconds",
-				  dict->path, diff);
-		}
-	}
-	return ret;
+	iov[0].iov_base = query;
+	iov[0].iov_len = strlen(query);
+	iov[1].iov_base = "\n";
+	iov[1].iov_len = 1;
+	ret = o_stream_sendv(dict->conn.conn.output, iov, 2);
+	if (ret < 0)
+		return -1;
+	i_assert((size_t)ret == iov[0].iov_len + 1);
+	return 0;
 }
 
-static int
-client_dict_read_one_line_real(struct client_dict *dict, char **line_r)
+static bool
+client_dict_cmd_send(struct client_dict *dict, struct client_dict_cmd **_cmd,
+		     const char **error_r)
 {
-	unsigned int id;
-	char *line;
-	ssize_t ret;
+	struct client_dict_cmd *cmd = *_cmd;
+	const char *error = NULL;
+	bool retry = cmd->retry_errors;
+	int ret;
 
-	*line_r = NULL;
-	while ((line = i_stream_next_line(dict->input)) == NULL) {
-		ret = client_dict_read_timeout(dict);
-		switch (ret) {
-		case -1:
-			if (dict->input->stream_errno != 0)
-				i_error("read(%s) failed: %m", dict->path);
-			else {
-				i_error("read(%s) failed: Remote disconnected",
-					dict->path);
-			}
-			return -1;
-		case -2:
-			i_error("read(%s) returned too much data", dict->path);
-			return -1;
-		case 0:
-			i_error("read(%s) failed: Timeout after %u seconds",
-				dict->path, DICT_CLIENT_READ_TIMEOUT_SECS);
-			return -1;
-		default:
-			i_assert(ret > 0);
-			break;
+	*_cmd = NULL;
+
+	/* we're no longer idling. even with no_replies=TRUE we're going to
+	   wait for COMMIT/ROLLBACK. */
+	if (dict->to_idle != NULL)
+		timeout_remove(&dict->to_idle);
+
+	if (client_dict_connect(dict, &error) < 0) {
+		retry = FALSE;
+		ret = -1;
+	} else {
+		ret = client_dict_cmd_query_send(dict, cmd->query);
+		if (ret < 0) {
+			error = t_strdup_printf("write(%s) failed: %s", dict->conn.conn.name,
+					o_stream_get_error(dict->conn.conn.output));
 		}
 	}
-	if (*line == DICT_PROTOCOL_REPLY_ASYNC_COMMIT) {
-		switch (line[1]) {
-		case DICT_PROTOCOL_REPLY_OK:
-			ret = 1;
-			break;
-		case DICT_PROTOCOL_REPLY_NOTFOUND:
+	if (ret < 0 && retry) {
+		/* Reconnect and try again. */
+		client_dict_disconnect(dict, error);
+		if (client_dict_connect(dict, &error) < 0)
+			;
+		else if (client_dict_cmd_query_send(dict, cmd->query) < 0) {
+			error = t_strdup_printf("write(%s) failed: %s", dict->conn.conn.name,
+				o_stream_get_error(dict->conn.conn.output));
+		} else {
 			ret = 0;
-			break;
-		case DICT_PROTOCOL_REPLY_FAIL:
-			ret = -1;
-			break;
-		default:
-			i_error("dict-client: Invalid async commit line: %s",
-				line);
-			return -1;
-		}
-		if (str_to_uint(line+2, &id) < 0) {
-			i_error("dict-client: Invalid ID");
-			return -1;
 		}
-		client_dict_finish_transaction(dict, id, ret);
-		return 0;
 	}
-	if (dict->iter_replies_skip > 0) {
-		/* called aborted the iteration before finishing it.
-		   skip over the iteration reply */
-		if (*line == DICT_PROTOCOL_REPLY_OK)
-			return 0;
-		if (*line != '\0' && *line != DICT_PROTOCOL_REPLY_FAIL) {
-			i_error("dict-client: Invalid iteration reply line: %s",
-				line);
-			return -1;
+
+	if (cmd->no_replies) {
+		/* just send and forget */
+		client_dict_cmd_unref(cmd);
+		return TRUE;
+	} else if (ret < 0) {
+		i_assert(error != NULL);
+		dict_cmd_callback_error(cmd, error);
+		client_dict_cmd_unref(cmd);
+		if (error_r != NULL)
+			*error_r = error;
+		return FALSE;
+	} else {
+		if (dict->to_requests == NULL) {
+			dict->to_requests =
+				timeout_add(DICT_CLIENT_REQUEST_TIMEOUT_MSECS,
+					    client_dict_input_timeout, dict);
 		}
-		dict->iter_replies_skip--;
-		return 0;
+		array_append(&dict->cmds, &cmd, 1);
+		return TRUE;
 	}
-	*line_r = line;
-	return 1;
 }
 
-static int client_dict_read_one_line(struct client_dict *dict, char **line_r)
+static void
+client_dict_transaction_send_begin(struct client_dict_transaction_context *ctx)
 {
-	int ret;
+	struct client_dict *dict = (struct client_dict *)ctx->ctx.dict;
+	struct client_dict_cmd *cmd;
+	const char *query, *error;
 
-	if ((ret = client_dict_read_one_line_real(dict, line_r)) < 0)
-		client_dict_disconnect(dict);
-	return ret;
+	i_assert(ctx->error == NULL);
+
+	ctx->sent_begin = TRUE;
+
+	/* transactions commands don't have replies. only COMMIT has. */
+	query = t_strdup_printf("%c%u", DICT_PROTOCOL_CMD_BEGIN, ctx->id);
+	cmd = client_dict_cmd_init(dict, query);
+	cmd->no_replies = TRUE;
+	cmd->retry_errors = TRUE;
+	if (!client_dict_cmd_send(dict, &cmd, &error))
+		ctx->error = i_strdup(error);
+}
+
+static void
+client_dict_send_transaction_query(struct client_dict_transaction_context *ctx,
+				   const char *query)
+{
+	struct client_dict *dict = (struct client_dict *)ctx->ctx.dict;
+	struct client_dict_cmd *cmd;
+	const char *error;
+
+	if (ctx->error != NULL)
+		return;
+
+	if (!ctx->sent_begin)
+		client_dict_transaction_send_begin(ctx);
+
+	cmd = client_dict_cmd_init(dict, query);
+	cmd->no_replies = TRUE;
+	if (!client_dict_cmd_send(dict, &cmd, &error))
+		ctx->error = i_strdup(error);
 }
 
 static bool client_dict_is_finished(struct client_dict *dict)
 {
-	return dict->transactions == NULL && !dict->in_iteration &&
-		dict->async_commits == 0;
+	return dict->transactions == NULL && array_count(&dict->cmds) == 0;
 }
 
 static void client_dict_timeout(struct client_dict *dict)
 {
 	if (client_dict_is_finished(dict))
-		client_dict_disconnect(dict);
+		client_dict_disconnect(dict, "Idle disconnection");
 }
 
 static void client_dict_add_timeout(struct client_dict *dict)
@@ -415,103 +318,153 @@ static void client_dict_add_timeout(struct client_dict *dict)
 	} else if (client_dict_is_finished(dict)) {
 		dict->to_idle = timeout_add(dict->idle_msecs,
 					    client_dict_timeout, dict);
+		if (dict->to_requests != NULL)
+			timeout_remove(&dict->to_requests);
 	}
 }
 
-static char *client_dict_read_line(struct client_dict *dict)
+static int dict_conn_input_line(struct connection *_conn, const char *line)
 {
-	char *line;
+	struct dict_connection *conn = (struct dict_connection *)_conn;
+	struct client_dict *dict = conn->dict;
+	struct client_dict_cmd *const *cmds;
+	unsigned int count;
+	bool finished;
+	int diff;
+
+	dict->last_input = ioloop_timeval;
+	if (dict->to_requests != NULL)
+		timeout_reset(dict->to_requests);
+
+	cmds = array_get(&conn->dict->cmds, &count);
+	if (count == 0) {
+		i_error("%s: Received reply without pending commands: %s",
+			dict->conn.conn.name, line);
+		return -1;
+	}
+	i_assert(!cmds[0]->no_replies);
 
-	while (client_dict_read_one_line(dict, &line) == 0)
-		;
+	client_dict_cmd_ref(cmds[0]);
+	finished = dict_cmd_callback_line(cmds[0], line);
+	if (!client_dict_cmd_unref(cmds[0])) {
+		/* disconnected during command handling */
+		return -1;
+	}
+	if (!finished) {
+		/* more lines needed for this command */
+		return 1;
+	}
+	diff = timeval_diff_msecs(&ioloop_timeval, &cmds[0]->start_time);
+	if (diff >= DICT_CLIENT_REQUEST_WARN_TIMEOUT_MSECS) {
+		i_warning("read(%s): dict lookup took %u.%03u seconds: %s",
+			  dict->conn.conn.name, diff/1000, diff % 1000,
+			  cmds[0]->query);
+	}
+	client_dict_cmd_unref(cmds[0]);
+	array_delete(&dict->cmds, 0, 1);
 
 	client_dict_add_timeout(dict);
-	return line;
+	return 1;
 }
 
-static int client_dict_connect(struct client_dict *dict)
+static int client_dict_connect(struct client_dict *dict, const char **error_r)
 {
 	const char *query;
 
+	if (dict->conn.conn.fd_in != -1)
+		return 0;
 	if (dict->last_failed_connect == ioloop_time) {
 		/* Try again later */
+		*error_r = "Waiting until the next connect attempt";
 		return -1;
 	}
 
-	dict->fd = net_connect_unix(dict->path);
-	if (dict->fd == -1) {
+	if (connection_client_connect(&dict->conn.conn) < 0) {
 		dict->last_failed_connect = ioloop_time;
 		if (errno == EACCES) {
-			i_error("%s", eacces_error_get("net_connect_unix",
-						       dict->path));
+			*error_r = eacces_error_get("net_connect_unix",
+						    dict->conn.conn.name);
 		} else {
-			i_error("net_connect_unix(%s) failed: %m",
-				dict->path);
+			*error_r = t_strdup_printf(
+				"net_connect_unix(%s) failed: %m", dict->conn.conn.name);
 		}
 		return -1;
 	}
 
-	/* Dictionary lookups are blocking */
-	net_set_nonblock(dict->fd, FALSE);
-
-	dict->input = i_stream_create_fd(dict->fd, (size_t)-1, FALSE);
-	dict->output = o_stream_create_fd(dict->fd, 4096, FALSE);
-
 	query = t_strdup_printf("%c%u\t%u\t%d\t%s\t%s\n",
 				DICT_PROTOCOL_CMD_HELLO,
 				DICT_CLIENT_PROTOCOL_MAJOR_VERSION,
 				DICT_CLIENT_PROTOCOL_MINOR_VERSION,
 				dict->value_type, dict->username, dict->uri);
-	if (client_dict_send_query(dict, query) < 0) {
-		dict->last_failed_connect = ioloop_time;
-		client_dict_disconnect(dict);
-		return -1;
-	}
-
-	dict->handshaked = TRUE;
+	o_stream_nsend_str(dict->conn.conn.output, query);
+	client_dict_add_timeout(dict);
 	return 0;
 }
 
-static void client_dict_disconnect(struct client_dict *dict)
+static void
+client_dict_abort_commands(struct client_dict *dict, const char *reason)
+{
+	ARRAY(struct client_dict_cmd *) cmds_copy;
+	struct client_dict_cmd *const *cmdp;
+
+	/* abort all commands */
+	t_array_init(&cmds_copy, array_count(&dict->cmds));
+	array_append_array(&cmds_copy, &dict->cmds);
+	array_clear(&dict->cmds);
+
+	array_foreach(&cmds_copy, cmdp) {
+		dict_cmd_callback_error(*cmdp, reason);
+		client_dict_cmd_unref(*cmdp);
+	}
+}
+
+static void client_dict_disconnect(struct client_dict *dict, const char *reason)
 {
 	struct client_dict_transaction_context *ctx, *next;
 
-	dict->connect_counter++;
-	dict->handshaked = FALSE;
-	dict->iter_replies_skip = 0;
+	client_dict_abort_commands(dict, reason);
 
-	/* abort all pending async commits */
+	/* all transactions that have sent BEGIN are no longer valid */
 	for (ctx = dict->transactions; ctx != NULL; ctx = next) {
 		next = ctx->next;
-		if (ctx->async)
-			client_dict_finish_transaction(dict, ctx->id, -1);
+		if (ctx->sent_begin && ctx->error == NULL)
+			ctx->error = i_strdup(reason);
 	}
 
 	if (dict->to_idle != NULL)
 		timeout_remove(&dict->to_idle);
-	if (dict->io != NULL)
-		io_remove(&dict->io);
-	if (dict->input != NULL)
-		i_stream_destroy(&dict->input);
-	if (dict->output != NULL)
-		o_stream_destroy(&dict->output);
+	if (dict->to_requests != NULL)
+		timeout_remove(&dict->to_requests);
+	connection_disconnect(&dict->conn.conn);
+}
 
-	if (dict->fd != -1) {
-		if (close(dict->fd) < 0)
-			i_error("close(%s) failed: %m", dict->path);
-		dict->fd = -1;
-	}
+static void dict_conn_destroy(struct connection *_conn)
+{
+	struct dict_connection *conn = (struct dict_connection *)_conn;
+
+	client_dict_disconnect(conn->dict, connection_disconnect_reason(_conn));
 }
 
+static const struct connection_settings dict_conn_set = {
+	.input_max_size = (size_t)-1,
+	.output_max_size = (size_t)-1,
+	.client = TRUE
+};
+
+static const struct connection_vfuncs dict_conn_vfuncs = {
+	.destroy = dict_conn_destroy,
+	.input_line = dict_conn_input_line
+};
+
 static int
 client_dict_init(struct dict *driver, const char *uri,
 		 const struct dict_settings *set,
 		 struct dict **dict_r, const char **error_r)
 {
+	struct ioloop *old_ioloop = current_ioloop;
 	struct client_dict *dict;
-	const char *p, *dest_uri;
+	const char *p, *dest_uri, *path;
 	unsigned int idle_msecs = DICT_CLIENT_DEFAULT_TIMEOUT_MSECS;
-	pool_t pool;
 
 	/* uri = [idle_msecs=<n>:] [<path>] ":" <uri> */
 	if (strncmp(uri, "idle_msecs=", 11) == 0) {
@@ -532,29 +485,36 @@ client_dict_init(struct dict *driver, const char *uri,
 		return -1;
 	}
 
-	pool = pool_alloconly_create("client dict", 1024);
-	dict = p_new(pool, struct client_dict, 1);
-	dict->pool = pool;
+	if (dict_connections == NULL) {
+		dict_connections = connection_list_init(&dict_conn_set,
+							&dict_conn_vfuncs);
+	}
+
+	dict = i_new(struct client_dict, 1);
 	dict->dict = *driver;
+	dict->conn.dict = dict;
 	dict->value_type = set->value_type;
-	dict->username = p_strdup(pool, set->username);
+	dict->username = i_strdup(set->username);
 	dict->idle_msecs = idle_msecs;
-
-	dict->fd = -1;
+	i_array_init(&dict->cmds, 32);
 
 	if (uri[0] == ':') {
 		/* default path */
-		dict->path = p_strconcat(pool, set->base_dir,
-				"/"DEFAULT_DICT_SERVER_SOCKET_FNAME, NULL);
+		path = t_strconcat(set->base_dir,
+			"/"DEFAULT_DICT_SERVER_SOCKET_FNAME, NULL);
 	} else if (uri[0] == '/') {
 		/* absolute path */
-		dict->path = p_strdup_until(pool, uri, dest_uri);
+		path = t_strdup_until(uri, dest_uri);
 	} else {
 		/* relative path to base_dir */
-		dict->path = p_strconcat(pool, set->base_dir, "/",
-				p_strdup_until(pool, uri, dest_uri), NULL);
+		path = t_strconcat(set->base_dir, "/",
+			t_strdup_until(uri, dest_uri), NULL);
 	}
-	dict->uri = p_strdup(pool, dest_uri + 1);
+	connection_init_client_unix(dict_connections, &dict->conn.conn, path);
+	dict->uri = i_strdup(dest_uri + 1);
+
+	dict->ioloop = io_loop_create();
+	io_loop_set_current(old_ioloop);
 	*dict_r = &dict->dict;
 	return 0;
 }
@@ -562,70 +522,234 @@ client_dict_init(struct dict *driver, const char *uri,
 static void client_dict_deinit(struct dict *_dict)
 {
 	struct client_dict *dict = (struct client_dict *)_dict;
+	struct ioloop *old_ioloop = current_ioloop;
+
+	client_dict_disconnect(dict, "Deinit");
+	connection_deinit(&dict->conn.conn);
 
-        client_dict_disconnect(dict);
 	i_assert(dict->transactions == NULL);
-	pool_unref(&dict->pool);
+	i_assert(array_count(&dict->cmds) == 0);
+
+	io_loop_set_current(dict->ioloop);
+	io_loop_destroy(&dict->ioloop);
+	io_loop_set_current(old_ioloop);
+
+	array_free(&dict->cmds);
+	i_free(dict->username);
+	i_free(dict->uri);
+	i_free(dict);
+
+	if (dict_connections->connections == NULL)
+		connection_list_deinit(&dict_connections);
 }
 
 static int client_dict_wait(struct dict *_dict)
 {
 	struct client_dict *dict = (struct client_dict *)_dict;
-	char *line;
-	int ret;
 
-	if (!dict->handshaked)
-		return -1;
+	if (array_count(&dict->cmds) == 0)
+		return 0;
 
-	while (dict->async_commits > 0) {
-		if ((ret = client_dict_read_one_line(dict, &line)) < 0)
-			return -1;
+	dict->prev_ioloop = current_ioloop;
+	io_loop_set_current(dict->ioloop);
 
-		if (ret > 0) {
-			i_error("dict-client: Unexpected reply waiting waiting for async commits: %s", line);
-			client_dict_disconnect(dict);
-			return -1;
-		}
-	}
+	if (dict->to_idle != NULL)
+		dict->to_idle = io_loop_move_timeout(&dict->to_idle);
+	if (dict->to_requests != NULL)
+		dict->to_requests = io_loop_move_timeout(&dict->to_requests);
+	connection_switch_ioloop(&dict->conn.conn);
+
+	while (array_count(&dict->cmds) > 0)
+		io_loop_run(dict->ioloop);
+
+	io_loop_set_current(dict->prev_ioloop);
+	dict->prev_ioloop = NULL;
+
+	if (dict->to_idle != NULL)
+		dict->to_idle = io_loop_move_timeout(&dict->to_idle);
+	if (dict->to_requests != NULL)
+		dict->to_requests = io_loop_move_timeout(&dict->to_requests);
+	connection_switch_ioloop(&dict->conn.conn);
 	return 0;
 }
 
-static int client_dict_lookup(struct dict *_dict, pool_t pool,
-			      const char *key, const char **value_r)
+static void
+client_dict_lookup_async_callback(struct client_dict_cmd *cmd, const char *line,
+				  const char *error)
+{
+	struct client_dict *dict = cmd->dict;
+	struct dict_lookup_result result;
+
+	memset(&result, 0, sizeof(result));
+	if (error != NULL) {
+		result.ret = -1;
+		result.error = error;
+	} else switch (*line) {
+	case DICT_PROTOCOL_REPLY_OK:
+		result.value = t_str_tabunescape(line + 1);
+		result.ret = 1;
+		break;
+	case DICT_PROTOCOL_REPLY_NOTFOUND:
+		result.ret = 0;
+		break;
+	case DICT_PROTOCOL_REPLY_FAIL:
+		result.error = line[1] == '\0' ? "dict-server returned failure" :
+			t_strdup_printf("dict-server returned failure: %s",
+			t_str_tabunescape(line+1));
+		result.ret = -1;
+		break;
+	default:
+		result.error = t_strdup_printf(
+			"dict-client: Invalid lookup '%s' reply: %s",
+			cmd->query, line);
+		client_dict_disconnect(dict, result.error);
+		result.ret = -1;
+		break;
+	}
+	dict_pre_api_callback(dict);
+	cmd->api_callback.lookup(&result, cmd->api_callback.context);
+	dict_post_api_callback(dict);
+}
+
+static void
+client_dict_lookup_async(struct dict *_dict, const char *key,
+			 dict_lookup_callback_t *callback, void *context)
 {
 	struct client_dict *dict = (struct client_dict *)_dict;
-	const char *line;
-	int ret;
+	struct client_dict_cmd *cmd;
+	const char *query;
 
-	T_BEGIN {
-		const char *query;
+	query = t_strdup_printf("%c%s", DICT_PROTOCOL_CMD_LOOKUP,
+				str_tabescape(key));
+	cmd = client_dict_cmd_init(dict, query);
+	cmd->callback = client_dict_lookup_async_callback;
+	cmd->api_callback.lookup = callback;
+	cmd->api_callback.context = context;
+	cmd->retry_errors = TRUE;
 
-		query = t_strdup_printf("%c%s\n", DICT_PROTOCOL_CMD_LOOKUP,
-					dict_client_escape(key));
-		ret = client_dict_send_query(dict, query);
-	} T_END;
-	if (ret < 0)
-		return -1;
+	client_dict_cmd_send(dict, &cmd, NULL);
+}
 
-	/* read reply */
-	line = client_dict_read_line(dict);
-	if (line == NULL)
-		return -1;
+static void client_dict_lookup_callback(const struct dict_lookup_result *result,
+					void *context)
+{
+	struct dict_lookup_result *result_copy = context;
 
-	switch (*line) {
-	case DICT_PROTOCOL_REPLY_OK:
-		*value_r = p_strdup(pool, dict_client_unescape(line + 1));
-		return 1;
-	case DICT_PROTOCOL_REPLY_NOTFOUND:
+	*result_copy = *result;
+}
+
+static int client_dict_lookup(struct dict *_dict, pool_t pool, const char *key,
+			      const char **value_r)
+{
+	struct dict_lookup_result result;
+
+	memset(&result, 0, sizeof(result));
+	result.ret = -2;
+
+	client_dict_lookup_async(_dict, key, client_dict_lookup_callback, &result);
+	if (result.ret == -2)
+		client_dict_wait(_dict);
+
+	switch (result.ret) {
+	case -1:
+		i_error("dict-client: Lookup '%s' failed: %s", key, result.error);
+		return -1;
+	case 0:
 		*value_r = NULL;
 		return 0;
+	case 1:
+		*value_r = p_strdup(pool, result.value);
+		return 1;
+	}
+	i_unreached();
+}
+
+static void client_dict_iterate_free(struct client_dict_iterate_context *ctx)
+{
+	if (!ctx->deinit || !ctx->finished)
+		return;
+	i_free(ctx->error);
+	i_free(ctx);
+}
+
+static void
+client_dict_iter_api_callback(struct client_dict_iterate_context *ctx,
+			      struct client_dict *dict)
+{
+	if (ctx->deinit) {
+		/* iterator was already deinitialized */
+		return;
+	}
+	if (ctx->ctx.async_callback != NULL) {
+		dict_pre_api_callback(dict);
+		ctx->ctx.async_callback(ctx->ctx.async_context);
+		dict_post_api_callback(dict);
+	} else {
+		/* synchronous lookup */
+		io_loop_stop(dict->ioloop);
+	}
+}
+
+static void
+client_dict_iter_async_callback(struct client_dict_cmd *cmd, const char *line,
+				const char *error)
+{
+	struct client_dict_iterate_context *ctx = cmd->iter;
+	struct client_dict *dict = cmd->dict;
+	struct client_dict_iter_result *result;
+	const char *key = NULL, *value = NULL;
+
+	if (error != NULL) {
+		/* failed */
+	} else switch (*line) {
+	case '\0':
+		/* end of iteration */
+		ctx->finished = TRUE;
+		client_dict_iter_api_callback(ctx, dict);
+		client_dict_iterate_free(ctx);
+		return;
+	case DICT_PROTOCOL_REPLY_OK:
+		/* key \t value */
+		key = line+1;
+		value = strchr(key, '\t');
+		break;
 	case DICT_PROTOCOL_REPLY_FAIL:
-		return -1;
+		error = t_strdup_printf("dict-server returned failure: %s", line+1);
+		break;
 	default:
-		i_error("dict-client: Invalid lookup '%s' reply: %s", key, line);
-		client_dict_disconnect(dict);
-		return -1;
+		break;
+	}
+	if (value == NULL && error == NULL) {
+		/* broken protocol */
+		error = t_strdup_printf("dict client (%s) sent broken iterate reply: %s",
+					dict->conn.conn.name, line);
+		client_dict_disconnect(dict, error);
+	}
+
+	if (error != NULL) {
+		if (ctx->error == NULL)
+			ctx->error = i_strdup(error);
+		ctx->finished = TRUE;
+		if (dict->prev_ioloop != NULL) {
+			/* stop client_dict_wait() */
+			io_loop_stop(dict->ioloop);
+		}
+		client_dict_iterate_free(ctx);
+		return;
+	}
+	cmd->unfinished = TRUE;
+
+	if (ctx->deinit) {
+		/* iterator was already deinitialized */
+		return;
 	}
+
+	key = t_strdup_until(key, value++);
+	result = array_append_space(&ctx->results);
+	result->key = p_strdup(ctx->results_pool, t_str_tabunescape(key));
+	result->value = p_strdup(ctx->results_pool, t_str_tabunescape(value));
+
+	client_dict_iter_api_callback(ctx, dict);
 }
 
 static struct dict_iterate_context *
@@ -634,28 +758,28 @@ client_dict_iterate_init(struct dict *_dict, const char *const *paths,
 {
 	struct client_dict *dict = (struct client_dict *)_dict;
         struct client_dict_iterate_context *ctx;
-
-	if (dict->in_iteration)
-		i_panic("dict-client: Only one iteration supported");
-	dict->in_iteration = TRUE;
+	struct client_dict_cmd *cmd;
+	string_t *query = t_str_new(256);
+	unsigned int i;
 
 	ctx = i_new(struct client_dict_iterate_context, 1);
 	ctx->ctx.dict = _dict;
-	ctx->pool = pool_alloconly_create("client dict iteration", 512);
+	ctx->results_pool = pool_alloconly_create("client dict iteration", 512);
+	ctx->async = (flags & DICT_ITERATE_FLAG_ASYNC) != 0;
+	i_array_init(&ctx->results, 64);
+
+	str_printfa(query, "%c%d", DICT_PROTOCOL_CMD_ITERATE, flags);
+	for (i = 0; paths[i] != NULL; i++) {
+		str_append_c(query, '\t');
+			str_append(query, str_tabescape(paths[i]));
+	}
 
-	T_BEGIN {
-		string_t *query = t_str_new(256);
-		unsigned int i;
+	cmd = client_dict_cmd_init(dict, str_c(query));
+	cmd->iter = ctx;
+	cmd->callback = client_dict_iter_async_callback;
+	cmd->retry_errors = TRUE;
 
-		str_printfa(query, "%c%d", DICT_PROTOCOL_CMD_ITERATE, flags);
-		for (i = 0; paths[i] != NULL; i++) {
-			str_append_c(query, '\t');
-			str_append(query, dict_client_escape(paths[i]));
-		}
-		str_append_c(query, '\n');
-		if (client_dict_send_query(dict, str_c(query)) < 0)
-			ctx->failed = TRUE;
-	} T_END;
+	client_dict_cmd_send(dict, &cmd, NULL);
 	return &ctx->ctx;
 }
 
@@ -664,52 +788,32 @@ static bool client_dict_iterate(struct dict_iterate_context *_ctx,
 {
 	struct client_dict_iterate_context *ctx =
 		(struct client_dict_iterate_context *)_ctx;
-	struct client_dict *dict = (struct client_dict *)_ctx->dict;
-	char *line, *key, *value;
-
-	if (ctx->failed)
-		return FALSE;
+	const struct client_dict_iter_result *results;
+	unsigned int count;
 
-	/* read next reply */
-	line = client_dict_read_line(dict);
-	if (line == NULL) {
-		ctx->failed = TRUE;
+	if (ctx->error != NULL) {
+		ctx->ctx.has_more = FALSE;
 		return FALSE;
 	}
 
-	if (*line == '\0') {
-		/* end of iteration */
-		ctx->finished = TRUE;
-		return FALSE;
+	results = array_get(&ctx->results, &count);
+	if (ctx->result_idx < count) {
+		*key_r = results[ctx->result_idx].key;
+		*value_r = results[ctx->result_idx].value;
+		ctx->ctx.has_more = TRUE;
+		ctx->result_idx++;
+		return TRUE;
 	}
-
-	/* line contains key \t value */
-	p_clear(ctx->pool);
-
-	switch (*line) {
-	case DICT_PROTOCOL_REPLY_OK:
-		key = line+1;
-		value = strchr(key, '\t');
-		break;
-	case DICT_PROTOCOL_REPLY_FAIL:
-		ctx->failed = TRUE;
-		return FALSE;
-	default:
-		key = NULL;
-		value = NULL;
-		break;
+	ctx->ctx.has_more = !ctx->finished;
+	ctx->result_idx = 0;
+	array_clear(&ctx->results);
+	p_clear(ctx->results_pool);
+
+	if (!ctx->async && ctx->ctx.has_more) {
+		client_dict_wait(_ctx->dict);
+		return client_dict_iterate(_ctx, key_r, value_r);
 	}
-	if (value == NULL) {
-		/* broken protocol */
-		i_error("dict client (%s) sent broken iterate reply: %s", dict->path, line);
-		ctx->failed = TRUE;
-		return FALSE;
-	}
-	*value++ = '\0';
-
-	*key_r = p_strdup(ctx->pool, dict_client_unescape(key));
-	*value_r = p_strdup(ctx->pool, dict_client_unescape(value));
-	return TRUE;
+	return FALSE;
 }
 
 static int client_dict_iterate_deinit(struct dict_iterate_context *_ctx)
@@ -717,14 +821,15 @@ static int client_dict_iterate_deinit(struct dict_iterate_context *_ctx)
 	struct client_dict *dict = (struct client_dict *)_ctx->dict;
 	struct client_dict_iterate_context *ctx =
 		(struct client_dict_iterate_context *)_ctx;
-	int ret = ctx->failed ? -1 : 0;
+	int ret = ctx->error != NULL ? -1 : 0;
 
-	if (!ctx->finished)
-		dict->iter_replies_skip++;
+	ctx->deinit = TRUE;
 
-	pool_unref(&ctx->pool);
-	i_free(ctx);
-	dict->in_iteration = FALSE;
+	if (ret < 0)
+		i_error("dict-client: Iteration failed: %s", ctx->error);
+	array_free(&ctx->results);
+	pool_unref(&ctx->results_pool);
+	client_dict_iterate_free(ctx);
 
 	client_dict_add_timeout(dict);
 	return ret;
@@ -744,23 +849,46 @@ client_dict_transaction_init(struct dict *_dict)
 	return &ctx->ctx;
 }
 
-static void dict_async_input(struct client_dict *dict)
+static void
+client_dict_transaction_commit_callback(struct client_dict_cmd *cmd,
+					const char *line, const char *error)
 {
-	char *line;
-	int ret;
-
-	i_assert(!dict->in_iteration);
+	struct client_dict *dict = cmd->dict;
+	int ret = -1;
 
-	do {
-		ret = client_dict_read_one_line(dict, &line);
-	} while (ret == 0 && i_stream_get_data_size(dict->input) > 0);
+	if (error != NULL) {
+		/* failed */
+		i_error("dict-client: Commit failed: %s", error);
+	} else switch (*line) {
+	case DICT_PROTOCOL_REPLY_OK:
+		ret = 1;
+		break;
+	case DICT_PROTOCOL_REPLY_NOTFOUND:
+		ret = 0;
+		break;
+	case DICT_PROTOCOL_REPLY_FAIL: {
+		const char *error = strchr(line+1, '\t');
 
-	if (ret < 0)
-		io_remove(&dict->io);
-	else if (ret > 0) {
-		i_error("dict-client: Unexpected reply waiting waiting for async commits: %s", line);
-		client_dict_disconnect(dict);
+		i_error("dict-client: server returned failure: %s",
+			error != NULL ? t_str_tabunescape(error) : "");
+		break;
+	}
+	default:
+		ret = -1;
+		error = t_strdup_printf("dict-client: Invalid commit reply: %s", line);
+		i_error("%s", error);
+		client_dict_disconnect(dict, error);
+		break;
 	}
+	dict_pre_api_callback(dict);
+	cmd->api_callback.commit(ret, cmd->api_callback.context);
+	dict_post_api_callback(dict);
+}
+
+static void commit_sync_callback(int ret, void *context)
+{
+	int *ret_p = context;
+	*ret_p = ret;
 }
 
 static int
@@ -772,65 +900,43 @@ client_dict_transaction_commit(struct dict_transaction_context *_ctx,
 	struct client_dict_transaction_context *ctx =
 		(struct client_dict_transaction_context *)_ctx;
 	struct client_dict *dict = (struct client_dict *)_ctx->dict;
-	unsigned int id;
-	int ret = ctx->failed ? -1 : 1;
+	struct client_dict_cmd *cmd;
+	const char *query;
+	int ret = -1;
 
-	ctx->committed = TRUE;
-	if (ctx->sent_begin && !ctx->failed) T_BEGIN {
-		const char *query, *line;
+	DLLIST_REMOVE(&dict->transactions, ctx);
 
-		query = t_strdup_printf("%c%u\n", !async ?
-					DICT_PROTOCOL_CMD_COMMIT :
-					DICT_PROTOCOL_CMD_COMMIT_ASYNC,
-					ctx->id);
-		if (client_dict_send_transaction_query(ctx, query) < 0)
-			ret = -1;
-		else if (async) {
-			ctx->callback = callback;
-			ctx->context = context;
-			ctx->async = TRUE;
-			if (dict->async_commits++ == 0) {
-				dict->io = io_add(dict->fd, IO_READ,
-						  dict_async_input, dict);
-			}
+	if (ctx->sent_begin && ctx->error == NULL) {
+		query = t_strdup_printf("%c%u", DICT_PROTOCOL_CMD_COMMIT, ctx->id);
+		cmd = client_dict_cmd_init(dict, query);
+		cmd->callback = client_dict_transaction_commit_callback;
+		if (callback != NULL) {
+			cmd->api_callback.commit = callback;
+			cmd->api_callback.context = context;
 		} else {
-			/* sync commit, read reply */
-			line = client_dict_read_line(dict);
-			if (line == NULL)
-				ret = -1;
-			else switch (*line) {
-			case DICT_PROTOCOL_REPLY_OK:
-				ret = 1;
-				break;
-			case DICT_PROTOCOL_REPLY_NOTFOUND:
-				ret = 0;
-				break;
-			case DICT_PROTOCOL_REPLY_FAIL:
-				ret = -1;
-				break;
-			default:
-				i_error("dict-client: Invalid commit reply: %s", line);
-				client_dict_disconnect(dict);
-				line = NULL;
-				ret = -1;
-				break;
-			}
-			if (line != NULL &&
-			    (str_to_uint(line+1, &id) < 0 || ctx->id != id)) {
-				i_error("dict-client: Invalid commit reply, "
-					"expected id=%u: %s", ctx->id, line);
-				client_dict_disconnect(dict);
-				ret = -1;
-			}
+			cmd->api_callback.commit = commit_sync_callback;
+			cmd->api_callback.context = &ret;
+		}
+		if (client_dict_cmd_send(dict, &cmd, NULL)) {
+			if (!async)
+				client_dict_wait(_ctx->dict);
 		}
-	} T_END;
+	} else if (ctx->error != NULL) {
+		/* already failed */
+		if (callback != NULL)
+			callback(-1, context);
+		ret = -1;
+	} else {
+		/* nothing changed */
+		if (callback != NULL)
+			callback(1, context);
+		ret = 1;
+	}
 
-	if (ret < 0 || !async) {
-		DLLIST_REMOVE(&dict->transactions, ctx);
-		i_free(ctx);
+	i_free(ctx->error);
+	i_free(ctx);
 
-		client_dict_add_timeout(dict);
-	}
+	client_dict_add_timeout(dict);
 	return ret;
 }
 
@@ -841,13 +947,13 @@ client_dict_transaction_rollback(struct dict_transaction_context *_ctx)
 		(struct client_dict_transaction_context *)_ctx;
 	struct client_dict *dict = (struct client_dict *)_ctx->dict;
 
-	if (ctx->sent_begin) T_BEGIN {
+	if (ctx->sent_begin) {
 		const char *query;
 
-		query = t_strdup_printf("%c%u\n", DICT_PROTOCOL_CMD_ROLLBACK,
+		query = t_strdup_printf("%c%u", DICT_PROTOCOL_CMD_ROLLBACK,
 					ctx->id);
 		client_dict_send_transaction_query(ctx, query);
-	} T_END;
+	}
 
 	DLLIST_REMOVE(&dict->transactions, ctx);
 	i_free(ctx);
@@ -860,16 +966,13 @@ static void client_dict_set(struct dict_transaction_context *_ctx,
 {
 	struct client_dict_transaction_context *ctx =
 		(struct client_dict_transaction_context *)_ctx;
+	const char *query;
 
-	T_BEGIN {
-		const char *query;
-
-		query = t_strdup_printf("%c%u\t%s\t%s\n",
-					DICT_PROTOCOL_CMD_SET, ctx->id,
-					dict_client_escape(key),
-					dict_client_escape(value));
-		client_dict_send_transaction_query(ctx, query);
-	} T_END;
+	query = t_strdup_printf("%c%u\t%s\t%s",
+				DICT_PROTOCOL_CMD_SET, ctx->id,
+				str_tabescape(key),
+				str_tabescape(value));
+	client_dict_send_transaction_query(ctx, query);
 }
 
 static void client_dict_unset(struct dict_transaction_context *_ctx,
@@ -877,32 +980,12 @@ static void client_dict_unset(struct dict_transaction_context *_ctx,
 {
 	struct client_dict_transaction_context *ctx =
 		(struct client_dict_transaction_context *)_ctx;
+	const char *query;
 
-	T_BEGIN {
-		const char *query;
-
-		query = t_strdup_printf("%c%u\t%s\n",
-					DICT_PROTOCOL_CMD_UNSET, ctx->id,
-					dict_client_escape(key));
-		client_dict_send_transaction_query(ctx, query);
-	} T_END;
-}
-
-static void client_dict_append(struct dict_transaction_context *_ctx,
-			       const char *key, const char *value)
-{
-	struct client_dict_transaction_context *ctx =
-		(struct client_dict_transaction_context *)_ctx;
-
-	T_BEGIN {
-		const char *query;
-
-		query = t_strdup_printf("%c%u\t%s\t%s\n",
-					DICT_PROTOCOL_CMD_APPEND, ctx->id,
-					dict_client_escape(key),
-					dict_client_escape(value));
-		client_dict_send_transaction_query(ctx, query);
-	} T_END;
+	query = t_strdup_printf("%c%u\t%s",
+				DICT_PROTOCOL_CMD_UNSET, ctx->id,
+				str_tabescape(key));
+	client_dict_send_transaction_query(ctx, query);
 }
 
 static void client_dict_atomic_inc(struct dict_transaction_context *_ctx,
@@ -910,14 +993,12 @@ static void client_dict_atomic_inc(struct dict_transaction_context *_ctx,
 {
 	struct client_dict_transaction_context *ctx =
 		(struct client_dict_transaction_context *)_ctx;
+	const char *query;
 
-	T_BEGIN {
-		const char *query;
-		query = t_strdup_printf("%c%u\t%s\t%lld\n",
-					DICT_PROTOCOL_CMD_ATOMIC_INC,
-					ctx->id, dict_client_escape(key), diff);
-		client_dict_send_transaction_query(ctx, query);
-	} T_END;
+	query = t_strdup_printf("%c%u\t%s\t%lld",
+				DICT_PROTOCOL_CMD_ATOMIC_INC,
+				ctx->id, str_tabescape(key), diff);
+	client_dict_send_transaction_query(ctx, query);
 }
 
 struct dict dict_driver_client = {
@@ -936,8 +1017,8 @@ struct dict dict_driver_client = {
 		client_dict_transaction_rollback,
 		client_dict_set,
 		client_dict_unset,
-		client_dict_append,
+		NULL,
 		client_dict_atomic_inc,
-		NULL
+		client_dict_lookup_async
 	}
 };

From adda15c338b012126164810dfbb2397b53d3d763 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 13 Jun 2016 17:10:22 +0300
Subject: [PATCH 313/450] lib-dict: Added dict_switch_ioloop()

---
 src/lib-dict/dict-client.c          | 21 ++++++++++++---------
 src/lib-dict/dict-file.c            |  1 +
 src/lib-dict/dict-memcached-ascii.c |  1 +
 src/lib-dict/dict-private.h         |  1 +
 src/lib-dict/dict-redis.c           |  1 +
 src/lib-dict/dict-sql.c             |  3 ++-
 src/lib-dict/dict.c                 |  8 ++++++++
 src/lib-dict/dict.h                 |  4 ++++
 src/plugins/dict-ldap/dict-ldap.c   | 15 ++++++++++++---
 9 files changed, 42 insertions(+), 13 deletions(-)

diff --git a/src/lib-dict/dict-client.c b/src/lib-dict/dict-client.c
index e1716b8f9cf..032ae0c3491 100644
--- a/src/lib-dict/dict-client.c
+++ b/src/lib-dict/dict-client.c
@@ -552,25 +552,27 @@ static int client_dict_wait(struct dict *_dict)
 
 	dict->prev_ioloop = current_ioloop;
 	io_loop_set_current(dict->ioloop);
-
-	if (dict->to_idle != NULL)
-		dict->to_idle = io_loop_move_timeout(&dict->to_idle);
-	if (dict->to_requests != NULL)
-		dict->to_requests = io_loop_move_timeout(&dict->to_requests);
-	connection_switch_ioloop(&dict->conn.conn);
-
+	dict_switch_ioloop(_dict);
 	while (array_count(&dict->cmds) > 0)
 		io_loop_run(dict->ioloop);
 
 	io_loop_set_current(dict->prev_ioloop);
 	dict->prev_ioloop = NULL;
 
+	dict_switch_ioloop(_dict);
+	return 0;
+}
+
+static bool client_dict_switch_ioloop(struct dict *_dict)
+{
+	struct client_dict *dict = (struct client_dict *)_dict;
+
 	if (dict->to_idle != NULL)
 		dict->to_idle = io_loop_move_timeout(&dict->to_idle);
 	if (dict->to_requests != NULL)
 		dict->to_requests = io_loop_move_timeout(&dict->to_requests);
 	connection_switch_ioloop(&dict->conn.conn);
-	return 0;
+	return array_count(&dict->cmds) > 0;
 }
 
 static void
@@ -1019,6 +1021,7 @@ struct dict dict_driver_client = {
 		client_dict_unset,
 		NULL,
 		client_dict_atomic_inc,
-		client_dict_lookup_async
+		client_dict_lookup_async,
+		client_dict_switch_ioloop
 	}
 };
diff --git a/src/lib-dict/dict-file.c b/src/lib-dict/dict-file.c
index b6d9f175064..e22465ee20d 100644
--- a/src/lib-dict/dict-file.c
+++ b/src/lib-dict/dict-file.c
@@ -663,6 +663,7 @@ struct dict dict_driver_file = {
 		dict_transaction_memory_unset,
 		dict_transaction_memory_append,
 		dict_transaction_memory_atomic_inc,
+		NULL,
 		NULL
 	}
 };
diff --git a/src/lib-dict/dict-memcached-ascii.c b/src/lib-dict/dict-memcached-ascii.c
index 7245a0229fb..ebb27871ba3 100644
--- a/src/lib-dict/dict-memcached-ascii.c
+++ b/src/lib-dict/dict-memcached-ascii.c
@@ -664,6 +664,7 @@ struct dict dict_driver_memcached_ascii = {
 		dict_transaction_memory_unset,
 		dict_transaction_memory_append,
 		dict_transaction_memory_atomic_inc,
+		NULL,
 		NULL
 	}
 };
diff --git a/src/lib-dict/dict-private.h b/src/lib-dict/dict-private.h
index 7fc0f00d2cf..f94e0552e75 100644
--- a/src/lib-dict/dict-private.h
+++ b/src/lib-dict/dict-private.h
@@ -38,6 +38,7 @@ struct dict_vfuncs {
 
 	void (*lookup_async)(struct dict *dict, const char *key,
 			     dict_lookup_callback_t *callback, void *context);
+	bool (*switch_ioloop)(struct dict *dict);
 };
 
 struct dict {
diff --git a/src/lib-dict/dict-redis.c b/src/lib-dict/dict-redis.c
index 8ff7160c3bb..b7e92ce9285 100644
--- a/src/lib-dict/dict-redis.c
+++ b/src/lib-dict/dict-redis.c
@@ -804,6 +804,7 @@ struct dict dict_driver_redis = {
 		redis_unset,
 		redis_append,
 		redis_atomic_inc,
+		NULL,
 		NULL
 	}
 };
diff --git a/src/lib-dict/dict-sql.c b/src/lib-dict/dict-sql.c
index f66d0a6047b..b142e360e18 100644
--- a/src/lib-dict/dict-sql.c
+++ b/src/lib-dict/dict-sql.c
@@ -1260,7 +1260,8 @@ static struct dict sql_dict = {
 		sql_dict_unset,
 		sql_dict_append,
 		sql_dict_atomic_inc,
-		sql_dict_lookup_async
+		sql_dict_lookup_async,
+		NULL
 	}
 };
 
diff --git a/src/lib-dict/dict.c b/src/lib-dict/dict.c
index e3d1deb2f5d..f6d3f68eead 100644
--- a/src/lib-dict/dict.c
+++ b/src/lib-dict/dict.c
@@ -106,6 +106,14 @@ int dict_wait(struct dict *dict)
 	return dict->v.wait == NULL ? 1 : dict->v.wait(dict);
 }
 
+bool dict_switch_ioloop(struct dict *dict)
+{
+	if (dict->v.switch_ioloop != NULL)
+		return dict->v.switch_ioloop(dict);
+	else
+		return FALSE;
+}
+
 static bool dict_key_prefix_is_valid(const char *key)
 {
 	return strncmp(key, DICT_PATH_SHARED, strlen(DICT_PATH_SHARED)) == 0 ||
diff --git a/src/lib-dict/dict.h b/src/lib-dict/dict.h
index ed661a9ac19..6188a1a1b71 100644
--- a/src/lib-dict/dict.h
+++ b/src/lib-dict/dict.h
@@ -70,6 +70,10 @@ void dict_deinit(struct dict **dict);
 /* Wait for all pending asynchronous operations to finish.
    Returns 0 if ok, -1 if error. */
 int dict_wait(struct dict *dict);
+/* Switch the dict to the current ioloop. This can be used to do dict_wait()
+   among other IO work. Returns TRUE if there is actually some work that can
+   be waited on. */
+bool dict_switch_ioloop(struct dict *dict) ATTR_NOWARN_UNUSED_RESULT;
 
 /* Lookup value for key. Set it to NULL if it's not found.
    Returns 1 if found, 0 if not found and -1 if lookup failed. */
diff --git a/src/plugins/dict-ldap/dict-ldap.c b/src/plugins/dict-ldap/dict-ldap.c
index 770e562e5f8..c241f63367c 100644
--- a/src/plugins/dict-ldap/dict-ldap.c
+++ b/src/plugins/dict-ldap/dict-ldap.c
@@ -240,14 +240,14 @@ int ldap_dict_wait(struct dict *dict) {
 
 	ctx->prev_ioloop = current_ioloop;
 	ctx->ioloop = io_loop_create();
-	ldap_client_switch_ioloop(ctx->client);
+	dict_switch_ioloop(dict);
 
 	do {
 		io_loop_run(current_ioloop);
 	} while (ctx->pending > 0);
 
 	io_loop_set_current(ctx->prev_ioloop);
-	ldap_client_switch_ioloop(ctx->client);
+	dict_switch_ioloop(dict);
 	io_loop_set_current(ctx->ioloop);
 	io_loop_destroy(&ctx->ioloop);
 	ctx->prev_ioloop = NULL;
@@ -255,6 +255,14 @@ int ldap_dict_wait(struct dict *dict) {
 	return 0;
 }
 
+static bool ldap_dict_switch_ioloop(struct dict *dict)
+{
+	struct ldap_dict *ctx = (struct ldap_dict *)dict;
+
+	ldap_client_switch_ioloop(ctx->client);
+	return ctx->pending > 0;
+}
+
 static
 void ldap_dict_lookup_done(const struct dict_lookup_result *result, void *ctx)
 {
@@ -429,7 +437,8 @@ struct dict dict_driver_ldap = {
 		NULL, /*ldap_unset,*/
 		NULL, /*ldap_append,*/
 		NULL, /*ldap_atomic_inc,*/
-		ldap_dict_lookup_async
+		ldap_dict_lookup_async,
+		ldap_dict_switch_ioloop
 	}
 };
 

From a5bf9e64b33bad8e589dff73907710837c186976 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 03:07:58 +0300
Subject: [PATCH 314/450] lib-dict-extra: Compiler warning fix

---
 src/lib-dict-extra/dict-fs.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lib-dict-extra/dict-fs.c b/src/lib-dict-extra/dict-fs.c
index ac951b9f579..3b4b58a79ed 100644
--- a/src/lib-dict-extra/dict-fs.c
+++ b/src/lib-dict-extra/dict-fs.c
@@ -285,6 +285,7 @@ struct dict dict_driver_fs = {
 		dict_transaction_memory_unset,
 		NULL,
 		NULL,
+		NULL,
 		NULL
 	}
 };

From e7a0878fd23bd175b66346a284ffc2c4f1d48fe9 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Sat, 21 May 2016 00:16:38 +0200
Subject: [PATCH 315/450] lib-http: response parser: Added check for the range
 of the response status value.

A value of 666 was accepted inappropriately.
---
 src/lib-http/http-response-parser.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/lib-http/http-response-parser.c b/src/lib-http/http-response-parser.c
index 807d75b8fb0..52aeda61f8d 100644
--- a/src/lib-http/http-response-parser.c
+++ b/src/lib-http/http-response-parser.c
@@ -73,6 +73,9 @@ static int http_response_parse_status(struct http_response_parser *parser)
 		return -1;
 	parser->response_status =
 		(p[0] - '0')*100 + (p[1] - '0')*10 + (p[2] - '0');
+	if (parser->response_status < 100 ||
+		parser->response_status >= 600)
+		return -1;
 	parser->parser.cur += 3;
 	return 1;
 }

From 4a4f676c4c9634abe1c111ea1ee621d07d808d5f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 18 May 2016 19:40:32 +0300
Subject: [PATCH 316/450] lib-http: Changed http_client_request_error to set
 request to NULL

It's going to internally unreference it, so the caller should be aware of it
also.

I also changed request state check to be an assert, since I don't think
there's any safe way this could work otherwise.
---
 src/lib-http/http-client-connection.c |  8 ++++----
 src/lib-http/http-client-host.c       |  2 +-
 src/lib-http/http-client-private.h    |  2 +-
 src/lib-http/http-client-queue.c      |  4 ++--
 src/lib-http/http-client-request.c    | 22 ++++++++++++----------
 5 files changed, 20 insertions(+), 18 deletions(-)

diff --git a/src/lib-http/http-client-connection.c b/src/lib-http/http-client-connection.c
index f725e5452b0..dfc5686b7e1 100644
--- a/src/lib-http/http-client-connection.c
+++ b/src/lib-http/http-client-connection.c
@@ -114,7 +114,7 @@ http_client_connection_abort_error(struct http_client_connection **_conn,
 
 	array_foreach_modifiable(&conn->request_wait_list, req) {
 		i_assert((*req)->submitted);
-		http_client_request_error(*req, status, error);
+		http_client_request_error(req, status, error);
 		http_client_request_unref(req);
 	}
 	array_clear(&conn->request_wait_list);
@@ -129,7 +129,7 @@ http_client_connection_abort_any_requests(struct http_client_connection *conn)
 	if (array_is_created(&conn->request_wait_list)) {
 		array_foreach_modifiable(&conn->request_wait_list, req) {
 			i_assert((*req)->submitted);
-			http_client_request_error(*req,
+			http_client_request_error(req,
 				HTTP_CLIENT_REQUEST_ERROR_ABORTED,
 				"Aborting");
 			http_client_request_unref(req);
@@ -139,7 +139,7 @@ http_client_connection_abort_any_requests(struct http_client_connection *conn)
 	if (conn->pending_request != NULL) {
 		struct http_client_request *pending_req = conn->pending_request;
 		conn->pending_request = NULL;
-		http_client_request_error(pending_req,
+		http_client_request_error(&pending_req,
 			HTTP_CLIENT_REQUEST_ERROR_ABORTED,
 			"Aborting");
 		http_client_request_unref(&pending_req);
@@ -819,7 +819,7 @@ static void http_client_connection_input(struct connection *_conn)
 			/* response cannot be 2xx if request payload was not completely sent
 			 */
 			if (early && response.status / 100 == 2) {
-				http_client_request_error(req,
+				http_client_request_error(&req,
 					HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE,
 					"Server responded with success response "
 					"before all payload was sent");
diff --git a/src/lib-http/http-client-host.c b/src/lib-http/http-client-host.c
index f0a0558e4fa..5c3c7d242c4 100644
--- a/src/lib-http/http-client-host.c
+++ b/src/lib-http/http-client-host.c
@@ -210,7 +210,7 @@ void http_client_host_submit_request(struct http_client_host *host,
 	if (http_client_peer_addr_is_https(&addr) &&
 		host->client->ssl_ctx == NULL) {
 		if (http_client_init_ssl_ctx(host->client, &error) < 0) {
-			http_client_request_error(req,
+			http_client_request_error(&req,
 				HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED, error);
 			return;
 		}
diff --git a/src/lib-http/http-client-private.h b/src/lib-http/http-client-private.h
index a5c0db766f2..9b1305360cb 100644
--- a/src/lib-http/http-client-private.h
+++ b/src/lib-http/http-client-private.h
@@ -296,7 +296,7 @@ void http_client_request_resubmit(struct http_client_request *req);
 void http_client_request_retry(struct http_client_request *req,
 	unsigned int status, const char *error);
 void http_client_request_error_delayed(struct http_client_request **_req);
-void http_client_request_error(struct http_client_request *req,
+void http_client_request_error(struct http_client_request **req,
 	unsigned int status, const char *error);
 void http_client_request_redirect(struct http_client_request *req,
 	unsigned int status, const char *location);
diff --git a/src/lib-http/http-client-queue.c b/src/lib-http/http-client-queue.c
index 153c80074ec..7b7e921b133 100644
--- a/src/lib-http/http-client-queue.c
+++ b/src/lib-http/http-client-queue.c
@@ -154,7 +154,7 @@ void http_client_queue_fail(struct http_client_queue *queue,
 	t_array_init(&treqs, array_count(req_arr));
 	array_copy(&treqs.arr, 0, &req_arr->arr, 0, array_count(req_arr));
 	array_foreach_modifiable(&treqs, req_idx) {
-		http_client_request_error(*req_idx, status, error);
+		http_client_request_error(req_idx, status, error);
 	}
 
 	/* all queues should be empty now... unless new requests were submitted
@@ -536,7 +536,7 @@ http_client_queue_request_timeout(struct http_client_queue *queue)
 
 		http_client_queue_debug(queue,
 			"Request %s timed out",	http_client_request_label(req));
-		http_client_request_error(req,
+		http_client_request_error(&req,
 			HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT,
 			"Timed out");
 	}
diff --git a/src/lib-http/http-client-request.c b/src/lib-http/http-client-request.c
index e5ce0083720..cec9748c60b 100644
--- a/src/lib-http/http-client-request.c
+++ b/src/lib-http/http-client-request.c
@@ -1088,11 +1088,12 @@ void http_client_request_error_delayed(struct http_client_request **_req)
 	http_client_request_destroy(&req);
 }
 
-void http_client_request_error(struct http_client_request *req,
+void http_client_request_error(struct http_client_request **_req,
 	unsigned int status, const char *error)
 {
-	if (req->state >= HTTP_REQUEST_STATE_FINISHED)
-		return;
+	struct http_client_request *req = *_req;
+
+	i_assert(req->state < HTTP_REQUEST_STATE_FINISHED);
 	req->state = HTTP_REQUEST_STATE_ABORTED;
 
 	if (req->queue != NULL)
@@ -1111,6 +1112,7 @@ void http_client_request_error(struct http_client_request *req,
 		http_client_request_send_error(req, status, error);
 		http_client_request_destroy(&req);
 	}
+	*_req = NULL;
 }
 
 void http_client_request_abort(struct http_client_request **_req)
@@ -1166,19 +1168,19 @@ void http_client_request_redirect(struct http_client_request *req,
 	/* parse URL */
 	if (http_url_parse(location, NULL, 0,
 			   pool_datastack_create(), &url, &error) < 0) {
-		http_client_request_error(req, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
+		http_client_request_error(&req, HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
 			t_strdup_printf("Invalid redirect location: %s", error));
 		return;
 	}
 
 	if (++req->redirects > req->client->set.max_redirects) {
 		if (req->client->set.max_redirects > 0) {
-			http_client_request_error(req,
+			http_client_request_error(&req,
 				HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
 				t_strdup_printf("Redirected more than %d times",
 					req->client->set.max_redirects));
 		} else {
-			http_client_request_error(req,
+			http_client_request_error(&req,
 				HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT,
 					"Redirect refused");
 		}
@@ -1189,7 +1191,7 @@ void http_client_request_redirect(struct http_client_request *req,
 	if (req->payload_input != NULL && req->payload_size > 0 && status != 303) {
 		if (req->payload_input->v_offset != req->payload_offset &&
 			!req->payload_input->seekable) {
-			http_client_request_error(req,
+			http_client_request_error(&req,
 				HTTP_CLIENT_REQUEST_ERROR_ABORTED,
 				"Redirect failed: Cannot resend payload; stream is not seekable");
 			return;
@@ -1252,7 +1254,7 @@ void http_client_request_resubmit(struct http_client_request *req)
 	if (req->payload_input != NULL && req->payload_size > 0) {
 		if (req->payload_input->v_offset != req->payload_offset &&
 			!req->payload_input->seekable) {
-			http_client_request_error(req,
+			http_client_request_error(&req,
 				HTTP_CLIENT_REQUEST_ERROR_ABORTED,
 				"Resubmission failed: Cannot resend payload; stream is not seekable");
 			return;
@@ -1265,7 +1267,7 @@ void http_client_request_resubmit(struct http_client_request *req)
 	if (req->payload_input != NULL && req->payload_size > 0) {
 		if (req->payload_input->v_offset != req->payload_offset &&
 			!req->payload_input->seekable) {
-			http_client_request_error(req,
+			http_client_request_error(&req,
 				HTTP_CLIENT_REQUEST_ERROR_ABORTED,
 				"Resubmission failed: Cannot resend payload; stream is not seekable");
 			return;
@@ -1288,7 +1290,7 @@ void http_client_request_retry(struct http_client_request *req,
 	unsigned int status, const char *error)
 {
 	if (!http_client_request_try_retry(req))
-		http_client_request_error(req, status, error);
+		http_client_request_error(&req, status, error);
 }
 
 bool http_client_request_try_retry(struct http_client_request *req)

From adc47546e129d4ded671dfb1bb7dc31c4823d5f8 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Thu, 19 May 2016 23:40:26 +0200
Subject: [PATCH 317/450] lib-http: client: Fixed reference counting for
 requests that are aborted due to having a broken outgoing payload stream.

---
 src/lib-http/http-client-request.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/lib-http/http-client-request.c b/src/lib-http/http-client-request.c
index cec9748c60b..05890f512bf 100644
--- a/src/lib-http/http-client-request.c
+++ b/src/lib-http/http-client-request.c
@@ -788,18 +788,18 @@ int http_client_request_send_more(struct http_client_request *req,
 	o_stream_set_max_buffer_size(output, (size_t)-1);
 
 	if (req->payload_input->stream_errno != 0) {
-		/* the payload stream assigned to this request is broken,
-		   fail this the request immediately */
-		http_client_request_send_error(req,
-			HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD,
-			"Broken payload stream");
-
 		/* we're in the middle of sending a request, so the connection
 		   will also have to be aborted */
 		errno = req->payload_input->stream_errno;
 		*error_r = t_strdup_printf("read(%s) failed: %s",
 					   i_stream_get_name(req->payload_input),
 					   i_stream_get_error(req->payload_input));
+		
+		/* the payload stream assigned to this request is broken,
+		   fail this the request immediately */
+		http_client_request_error(&req,
+			HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD,
+			"Broken payload stream");
 		return -1;
 	} else if (output->stream_errno != 0) {
 		/* failed to send request */

From 2cbef0faed1c4d3a99ab580ed4abf7344bb9fbf1 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Sun, 22 May 2016 10:42:01 +0200
Subject: [PATCH 318/450] lib-http: client: Fixed reference counting for
 requests that are aborted due to an early server response.

An additional http_client_request_unref() is (now) unneccessary.
---
 src/lib-http/http-client-connection.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/lib-http/http-client-connection.c b/src/lib-http/http-client-connection.c
index dfc5686b7e1..1e9be7f681c 100644
--- a/src/lib-http/http-client-connection.c
+++ b/src/lib-http/http-client-connection.c
@@ -823,7 +823,6 @@ static void http_client_connection_input(struct connection *_conn)
 					HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE,
 					"Server responded with success response "
 					"before all payload was sent");
-				http_client_request_unref(&req);
 				http_client_connection_close(&conn);
 				return;
 			} 

From d2b6ff054e61f86d93b00dc7fa36adaaa6dc711c Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Sat, 21 May 2016 13:16:08 +0200
Subject: [PATCH 319/450] lib-http: client: Improved request reference counting
 in connection code.

It should now always be clear when the connection object holds a reference to a request and when it is released.
Only while the reference is held, req->conn points to a connection.
This also makes the assertion in http_client_request_unref() more robust and clear.
---
 src/lib-http/http-client-connection.c | 117 +++++++++++++++++---------
 src/lib-http/http-client-request.c    |   5 +-
 2 files changed, 78 insertions(+), 44 deletions(-)

diff --git a/src/lib-http/http-client-connection.c b/src/lib-http/http-client-connection.c
index 1e9be7f681c..c25c903580e 100644
--- a/src/lib-http/http-client-connection.c
+++ b/src/lib-http/http-client-connection.c
@@ -68,15 +68,23 @@ static void
 http_client_connection_retry_requests(struct http_client_connection *conn,
 	unsigned int status, const char *error)
 {
-	struct http_client_request **req;
+	struct http_client_request *req, **req_idx;
 
 	if (!array_is_created(&conn->request_wait_list))
 		return;
 
-	array_foreach_modifiable(&conn->request_wait_list, req) {
-		if ((*req)->state < HTTP_REQUEST_STATE_FINISHED)
-			http_client_request_retry(*req, status, error);
-		http_client_request_unref(req);
+	http_client_connection_debug(conn,
+		"Retrying pending requests");
+
+	array_foreach_modifiable(&conn->request_wait_list, req_idx) {
+		req = *req_idx;
+		/* drop reference from connection */
+		req->conn = NULL;
+		if (!http_client_request_unref(req_idx))
+			continue;
+		/* retry the request, which may drop it */
+		if (req->state < HTTP_REQUEST_STATE_FINISHED)
+			http_client_request_retry(req, status, error);
 	}	
 	array_clear(&conn->request_wait_list);
 }
@@ -85,15 +93,20 @@ static void
 http_client_connection_server_close(struct http_client_connection **_conn)
 {
 	struct http_client_connection *conn = *_conn;
-	struct http_client_request **req;
+	struct http_client_request *req, **req_idx;
 
 	http_client_connection_debug(conn,
 		"Server explicitly closed connection");
 
-	array_foreach_modifiable(&conn->request_wait_list, req) {
-		if ((*req)->state < HTTP_REQUEST_STATE_FINISHED)
-			http_client_request_resubmit(*req);
-		http_client_request_unref(req);
+	array_foreach_modifiable(&conn->request_wait_list, req_idx) {
+		req = *req_idx;
+		/* drop reference from connection */
+		req->conn = NULL;
+		if (!http_client_request_unref(req_idx))
+			continue;
+		/* resubmit the request, which may drop it */
+		if ((*req_idx)->state < HTTP_REQUEST_STATE_FINISHED)
+			http_client_request_resubmit(req);
 	}	
 	array_clear(&conn->request_wait_list);
 
@@ -108,14 +121,19 @@ http_client_connection_abort_error(struct http_client_connection **_conn,
 	unsigned int status, const char *error)
 {
 	struct http_client_connection *conn = *_conn;
-	struct http_client_request **req;
+	struct http_client_request *req, **req_idx;
 
 	http_client_connection_debug(conn, "Aborting connection: %s", error);
 
-	array_foreach_modifiable(&conn->request_wait_list, req) {
-		i_assert((*req)->submitted);
-		http_client_request_error(req, status, error);
-		http_client_request_unref(req);
+	array_foreach_modifiable(&conn->request_wait_list, req_idx) {
+		req = *req_idx;
+		i_assert(req->submitted);
+		/* drop reference from connection */
+		req->conn = NULL;
+		if (!http_client_request_unref(req_idx))
+			continue;
+		/* drop request if not already aborted */
+		http_client_request_error(&req, status, error);
 	}
 	array_clear(&conn->request_wait_list);
 	http_client_connection_close(_conn);
@@ -124,25 +142,33 @@ http_client_connection_abort_error(struct http_client_connection **_conn,
 static void
 http_client_connection_abort_any_requests(struct http_client_connection *conn)
 {
-	struct http_client_request **req;
+	struct http_client_request *req, **req_idx;
 
 	if (array_is_created(&conn->request_wait_list)) {
-		array_foreach_modifiable(&conn->request_wait_list, req) {
-			i_assert((*req)->submitted);
-			http_client_request_error(req,
+		array_foreach_modifiable(&conn->request_wait_list, req_idx) {
+			req = *req_idx;
+			i_assert(req->submitted);
+			/* drop reference from connection */
+			req->conn = NULL;
+			if (!http_client_request_unref(req_idx))
+				continue;
+			/* drop request if not already aborted */
+			http_client_request_error(&req,
 				HTTP_CLIENT_REQUEST_ERROR_ABORTED,
 				"Aborting");
-			http_client_request_unref(req);
 		}
 		array_clear(&conn->request_wait_list);
 	}
 	if (conn->pending_request != NULL) {
-		struct http_client_request *pending_req = conn->pending_request;
-		conn->pending_request = NULL;
-		http_client_request_error(&pending_req,
-			HTTP_CLIENT_REQUEST_ERROR_ABORTED,
-			"Aborting");
-		http_client_request_unref(&pending_req);
+		req = conn->pending_request;
+		/* drop reference from connection */
+		req->conn = NULL;
+		if (http_client_request_unref(&conn->pending_request)) {
+			/* drop request if not already aborted */
+			http_client_request_error(&req,
+				HTTP_CLIENT_REQUEST_ERROR_ABORTED,
+				"Aborting");
+		}
 	}
 }
 
@@ -402,12 +428,13 @@ int http_client_connection_next_request(struct http_client_connection *conn)
 	if (conn->to_idle != NULL)
 		timeout_remove(&conn->to_idle);
 
-	req->conn = conn;
 	req->payload_sync_continue = FALSE;
 	if (conn->peer->no_payload_sync)
 		req->payload_sync = FALSE;
 
+	/* add request to wait list and add a reference */
 	array_append(&conn->request_wait_list, &req, 1);
+	req->conn = conn;
 	http_client_request_ref(req);
 
 	http_client_connection_debug(conn, "Claimed request %s",
@@ -519,15 +546,15 @@ static void http_client_payload_destroyed(struct http_client_request *req)
 	   the payload. make sure here that it's switched back. */
 	net_set_nonblock(conn->conn.fd_in, TRUE);
 
+	/* drop reference from connection */
 	req->conn = NULL;
+	if (http_client_request_unref(&conn->pending_request)) {
+		/* finish request if not already aborted */
+		http_client_request_finish(req);
+	}
+
 	conn->incoming_payload = NULL;
-	conn->pending_request = NULL;
 	http_client_connection_ref(conn);
-	http_client_request_finish(req);
-
-	/* room for new requests */
-	if (http_client_connection_is_ready(conn))
-		http_client_peer_trigger_request_handler(conn->peer);
 
 	/* input stream may have pending input. make sure input handler
 	   gets called (but don't do it directly, since we get get here
@@ -537,15 +564,20 @@ static void http_client_payload_destroyed(struct http_client_request *req)
 	conn->to_input =
 		timeout_add_short(0, http_client_payload_destroyed_timeout, conn);
 
-	http_client_request_unref(&req);
+	/* room for new requests */
+	if (http_client_connection_is_ready(conn))
+		http_client_peer_trigger_request_handler(conn->peer);
+
 	http_client_connection_unref(&conn);
 }
 
 static bool
-http_client_connection_return_response(struct http_client_request *req,
+http_client_connection_return_response(
+	struct http_client_connection *conn,
+	struct http_client_request *req,
 	struct http_response *response)
 {
-	struct http_client_connection *conn = req->conn, *tmp_conn;
+	struct http_client_connection *tmp_conn;
 	struct istream *payload;
 	bool retrying, ret;
 
@@ -579,7 +611,6 @@ http_client_connection_return_response(struct http_client_request *req,
 	tmp_conn = conn;
 	if (!http_client_connection_unref(&tmp_conn)) {
 		/* the callback managed to get this connection destroyed */
-		req->conn = NULL;
 		if (!retrying)
 			http_client_request_finish(req);
 		http_client_request_unref(&req);
@@ -606,6 +637,9 @@ http_client_connection_return_response(struct http_client_request *req,
 		req->state = HTTP_REQUEST_STATE_PAYLOAD_IN;
 		payload = response->payload;
 		response->payload = NULL;
+
+		/* maintain request reference while payload is pending */
+		req->conn = conn;
 		conn->pending_request = req;
 
 		/* request is dereferenced in payload destroy callback */
@@ -616,7 +650,6 @@ http_client_connection_return_response(struct http_client_request *req,
 			http_client_payload_finished(conn);
 		}
 	} else {
-		req->conn = NULL;
 		http_client_request_finish(req);
 		http_client_request_unref(&req);
 	}
@@ -804,6 +837,7 @@ static void http_client_connection_input(struct connection *_conn)
 
 		/* remove request from queue */
 		array_delete(&conn->request_wait_list, 0, 1);
+		req->conn = NULL;
 		aborted = (req->state == HTTP_REQUEST_STATE_ABORTED);
 		req_ref = req;
 		if (!http_client_request_unref(&req_ref)) {
@@ -870,8 +904,8 @@ static void http_client_connection_input(struct connection *_conn)
 
 			if (!handled) {
 				/* response for application */
-				i_assert(req->conn == conn);
-				if (!http_client_connection_return_response(req, &response))
+				if (!http_client_connection_return_response
+					(conn, req, &response))
 					return;
 			}
 		}
@@ -1056,14 +1090,13 @@ http_client_connection_ready(struct http_client_connection *conn)
 			struct http_response response;
 
 			http_client_request_ref(req);
-			req->conn = conn;
 			conn->tunneling = TRUE;
 
 			memset(&response, 0, sizeof(response));
 			response.status = 200;
 			response.reason = "OK";
 
-			(void)http_client_connection_return_response(req, &response);
+			(void)http_client_connection_return_response(conn, req, &response);
 			http_client_request_unref(&req);
 			return;
 		} 
diff --git a/src/lib-http/http-client-request.c b/src/lib-http/http-client-request.c
index 05890f512bf..85157625adf 100644
--- a/src/lib-http/http-client-request.c
+++ b/src/lib-http/http-client-request.c
@@ -172,7 +172,7 @@ bool http_client_request_unref(struct http_client_request **_req)
 		client->requests_count);
 
 	/* cannot be destroyed while it is still pending */
-	i_assert(req->conn == NULL || req->conn->pending_request == NULL);
+	i_assert(req->conn == NULL);
 
 	if (req->queue != NULL)
 		http_client_queue_drop_request(req->queue, req);
@@ -1144,7 +1144,8 @@ void http_client_request_finish(struct http_client_request *req)
 	if (req->state >= HTTP_REQUEST_STATE_FINISHED)
 		return;
 
-	i_assert(req->refcount > 1);
+	i_assert(req->refcount > 0);
+
 	http_client_request_debug(req, "Finished");
 
 	req->callback = NULL;

From 0102e440ac9c078f9358cd0c98d8a27f4d2edd10 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Sat, 21 May 2016 00:16:21 +0200
Subject: [PATCH 320/450] lib-http: client: Made peer object reference-counted
 to prevent invalid memory access in request handling routine.

Resetting the peer->handling_requests flag risked triggering a segfault, since the peer object could be deleted from within the request handler loop.
---
 src/lib-http/http-client-peer.c    | 97 +++++++++++++++++++-----------
 src/lib-http/http-client-private.h |  8 ++-
 src/lib-http/http-client.c         |  2 +-
 3 files changed, 70 insertions(+), 37 deletions(-)

diff --git a/src/lib-http/http-client-peer.c b/src/lib-http/http-client-peer.c
index ed4e305dd35..a615715499f 100644
--- a/src/lib-http/http-client-peer.c
+++ b/src/lib-http/http-client-peer.c
@@ -128,7 +128,7 @@ http_client_peer_connect_backoff(struct http_client_peer *peer)
 	timeout_remove(&peer->to_backoff);
 
 	if (array_count(&peer->queues) == 0) {
-		http_client_peer_free(&peer);
+		http_client_peer_close(&peer);
 		return;
 	}
 
@@ -183,23 +183,6 @@ bool http_client_peer_is_connected(struct http_client_peer *peer)
 	return FALSE;
 }
 
-static void
-http_client_peer_disconnect(struct http_client_peer *peer)
-{
-	struct http_client_connection **conn;
-	ARRAY_TYPE(http_client_connection) conns;
-
-	http_client_peer_debug(peer, "Peer disconnect");
-
-	/* make a copy of the connection array; freed connections modify it */
-	t_array_init(&conns, array_count(&peer->conns));
-	array_copy(&conns.arr, 0, &peer->conns.arr, 0, array_count(&peer->conns));
-	array_foreach_modifiable(&conns, conn) {
-		http_client_connection_unref(conn);
-	}
-	i_assert(array_count(&peer->conns) == 0);
-}
-
 static unsigned int
 http_client_peer_requests_pending(struct http_client_peer *peer,
 				  unsigned int *num_urgent_r)
@@ -225,7 +208,7 @@ static void http_client_peer_check_idle(struct http_client_peer *peer)
 	if (array_count(&peer->conns) == 0 &&
 		http_client_peer_requests_pending(peer, &num_urgent) == 0) {
 		/* no connections or pending requests; die immediately */
-		http_client_peer_free(&peer);
+		http_client_peer_close(&peer);
 		return;
 	}
 
@@ -247,6 +230,7 @@ http_client_peer_handle_requests_real(struct http_client_peer *peer)
 	struct _conn_available *conn_avail_idx;
 	unsigned int connecting, closing, idle;
 	unsigned int num_pending, num_urgent, new_connections, 	working_conn_count;
+	struct http_client_peer *tmp_peer;
 	bool statistics_dirty = TRUE;
 
 	/* FIXME: limit the number of requests handled in one run to prevent
@@ -258,7 +242,7 @@ http_client_peer_handle_requests_real(struct http_client_peer *peer)
 		http_client_peer_debug(peer,
 			"Peer no longer used; will now disconnect "
 			"(%u connections exist)", array_count(&peer->conns));
-		http_client_peer_disconnect(peer);
+		http_client_peer_close(&peer);
 		return;
 	}
 
@@ -272,6 +256,7 @@ http_client_peer_handle_requests_real(struct http_client_peer *peer)
 		return;
 	}
 
+	http_client_peer_ref(peer);
 	peer->handling_requests = TRUE;
 	t_array_init(&conns_avail, array_count(&peer->conns));
 	do {
@@ -364,6 +349,10 @@ http_client_peer_handle_requests_real(struct http_client_peer *peer)
 			break;
 		}
 	} while (statistics_dirty);
+
+	tmp_peer = peer;
+	if (!http_client_peer_unref(&tmp_peer))
+		return;
 	peer->handling_requests = FALSE;
 
 	if (num_pending == 0)
@@ -494,6 +483,7 @@ http_client_peer_create(struct http_client *client,
 	struct http_client_peer *peer;
 
 	peer = i_new(struct http_client_peer, 1);
+	peer->refcount = 1;
 	peer->client = client;
 	peer->addr = *addr;
 
@@ -528,33 +518,72 @@ http_client_peer_create(struct http_client *client,
 	return peer;
 }
 
-void http_client_peer_free(struct http_client_peer **_peer)
+static void
+http_client_peer_disconnect(struct http_client_peer *peer)
 {
-	struct http_client_peer *peer = *_peer;
-
-	*_peer = NULL;
+	struct http_client_connection **conn;
+	ARRAY_TYPE(http_client_connection) conns;
 
-	if (peer->destroyed)
+	if (peer->disconnected)
 		return;
-	peer->destroyed = TRUE;
+	peer->disconnected = TRUE;
 
-	http_client_peer_debug(peer, "Peer destroy");
+	http_client_peer_debug(peer, "Peer disconnect");
+
+	/* make a copy of the connection array; freed connections modify it */
+	t_array_init(&conns, array_count(&peer->conns));
+	array_copy(&conns.arr, 0, &peer->conns.arr, 0, array_count(&peer->conns));
+	array_foreach_modifiable(&conns, conn) {
+		http_client_connection_unref(conn);
+	}
+	i_assert(array_count(&peer->conns) == 0);
 
 	if (peer->to_req_handling != NULL)
 		timeout_remove(&peer->to_req_handling);
 	if (peer->to_backoff != NULL)
 		timeout_remove(&peer->to_backoff);
 
-	http_client_peer_disconnect(peer);
-	array_free(&peer->conns);
-	array_free(&peer->queues);
-
 	hash_table_remove
 		(peer->client->peers, (const struct http_client_peer_addr *)&peer->addr);
 	DLLIST_REMOVE(&peer->client->peers_list, peer);
+}
 
+void http_client_peer_ref(struct http_client_peer *peer)
+{
+	peer->refcount++;
+}
+
+bool http_client_peer_unref(struct http_client_peer **_peer)
+{
+	struct http_client_peer *peer = *_peer;
+
+	i_assert(peer->refcount > 0);
+
+	*_peer = NULL;
+
+	if (--peer->refcount > 0)
+		return TRUE;
+
+	http_client_peer_debug(peer, "Peer destroy");
+
+	http_client_peer_disconnect(peer);
+
+	array_free(&peer->conns);
+	array_free(&peer->queues);
 	i_free(peer->addr_name);
 	i_free(peer);
+	return FALSE;
+}
+
+void http_client_peer_close(struct http_client_peer **_peer)
+{
+	struct http_client_peer *peer = *_peer;
+
+	http_client_peer_debug(peer, "Peer close");
+
+	http_client_peer_disconnect(peer);
+
+	(void)http_client_peer_unref(_peer);
 }
 
 struct http_client_peer *
@@ -604,7 +633,7 @@ void http_client_peer_unlink_queue(struct http_client_peer *peer,
 					http_client_peer_trigger_request_handler(peer);
 				} else {
 					/* drop peer immediately */
-					http_client_peer_free(&peer);
+					http_client_peer_close(&peer);
 				}
 			}
 			return;
@@ -702,7 +731,7 @@ void http_client_peer_connection_lost(struct http_client_peer *peer)
 	   connect itself fails, http_client_peer_connection_failure() is
 	   called instead. */
 
-	if (peer->destroyed)
+	if (peer->disconnected)
 		return;
 
 	http_client_peer_debug(peer, "Lost a connection (%d connections left)",
@@ -716,7 +745,7 @@ void http_client_peer_connection_lost(struct http_client_peer *peer)
 	/* check if peer is still relevant */
 	if (array_count(&peer->conns) == 0 &&
 		http_client_peer_requests_pending(peer, &num_urgent) == 0) {
-		http_client_peer_free(&peer);
+		http_client_peer_close(&peer);
 		return;
 	}
 
diff --git a/src/lib-http/http-client-private.h b/src/lib-http/http-client-private.h
index 9b1305360cb..9f8e81eecbd 100644
--- a/src/lib-http/http-client-private.h
+++ b/src/lib-http/http-client-private.h
@@ -165,6 +165,7 @@ struct http_client_connection {
 };
 
 struct http_client_peer {
+	unsigned int refcount;
 	struct http_client_peer_addr addr;
 	char *addr_name;
 
@@ -185,7 +186,7 @@ struct http_client_peer {
 	struct timeout *to_backoff;
 	unsigned int backoff_time_msecs;
 
-	unsigned int destroyed:1;        /* peer is being destroyed */
+	unsigned int disconnected:1;     /* peer is already disconnected */
 	unsigned int no_payload_sync:1;  /* expect: 100-continue failed before */
 	unsigned int seen_100_response:1;/* expect: 100-continue succeeded before */
 	unsigned int allows_pipelining:1;/* peer is known to allow persistent
@@ -337,7 +338,10 @@ int http_client_peer_addr_cmp
 struct http_client_peer *
 	http_client_peer_get(struct http_client *client,
 		const struct http_client_peer_addr *addr);
-void http_client_peer_free(struct http_client_peer **_peer);
+void http_client_peer_ref(struct http_client_peer *peer);
+bool http_client_peer_unref(struct http_client_peer **_peer);
+void http_client_peer_close(struct http_client_peer **_peer);
+
 bool http_client_peer_have_queue(struct http_client_peer *peer,
 				struct http_client_queue *queue);
 void http_client_peer_link_queue(struct http_client_peer *peer,
diff --git a/src/lib-http/http-client.c b/src/lib-http/http-client.c
index 4ed33452634..7b034428c1b 100644
--- a/src/lib-http/http-client.c
+++ b/src/lib-http/http-client.c
@@ -187,7 +187,7 @@ void http_client_deinit(struct http_client **_client)
 	/* free peers */
 	while (client->peers_list != NULL) {
 		peer = client->peers_list;
-		http_client_peer_free(&peer);
+		http_client_peer_close(&peer);
 	}
 	hash_table_destroy(&client->peers);
 

From 8d5c693b138715098a8cc43e5226c795ae2461ad Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Sat, 21 May 2016 13:17:58 +0200
Subject: [PATCH 321/450] lib-http: client: peer: Improved debug message that
 shows the loss of a connection.

---
 src/lib-http/http-client-peer.c | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/lib-http/http-client-peer.c b/src/lib-http/http-client-peer.c
index a615715499f..1f67899bf44 100644
--- a/src/lib-http/http-client-peer.c
+++ b/src/lib-http/http-client-peer.c
@@ -725,7 +725,7 @@ void http_client_peer_connection_failure(struct http_client_peer *peer,
 
 void http_client_peer_connection_lost(struct http_client_peer *peer)
 {
-	unsigned int num_urgent;
+	unsigned int num_pending, num_urgent;
 
 	/* we get here when an already connected connection fails. if the
 	   connect itself fails, http_client_peer_connection_failure() is
@@ -734,8 +734,12 @@ void http_client_peer_connection_lost(struct http_client_peer *peer)
 	if (peer->disconnected)
 		return;
 
-	http_client_peer_debug(peer, "Lost a connection (%d connections left)",
-		array_count(&peer->conns));
+	num_pending = http_client_peer_requests_pending(peer, &num_urgent);
+
+	http_client_peer_debug(peer,
+		"Lost a connection "
+		"(%d connections left, %u requests pending, %u requests urgent)",
+		array_count(&peer->conns), num_pending, num_urgent);
 
 	if (peer->handling_requests) {
 		/* we got here from the request handler loop */

From d63cb134209aa5c4313cb8f0730722103fc45826 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Thu, 19 May 2016 23:42:14 +0200
Subject: [PATCH 322/450] lib-http: client: Removed curiously duplicated code.

Probably a patch got applied with sufficient fuzz that it duplicated this code fragment, while it actually already existed.
---
 src/lib-http/http-client-request.c | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/src/lib-http/http-client-request.c b/src/lib-http/http-client-request.c
index 85157625adf..0c1a7013639 100644
--- a/src/lib-http/http-client-request.c
+++ b/src/lib-http/http-client-request.c
@@ -1264,19 +1264,6 @@ void http_client_request_resubmit(struct http_client_request *req)
 		}
 	}
 
-	/* rewind payload stream */
-	if (req->payload_input != NULL && req->payload_size > 0) {
-		if (req->payload_input->v_offset != req->payload_offset &&
-			!req->payload_input->seekable) {
-			http_client_request_error(&req,
-				HTTP_CLIENT_REQUEST_ERROR_ABORTED,
-				"Resubmission failed: Cannot resend payload; stream is not seekable");
-			return;
-		} else {
-			i_stream_seek(req->payload_input, req->payload_offset);
-		}
-	}
-
 	/* drop payload output stream from previous attempt */
 	if (req->payload_output != NULL)
 		o_stream_unref(&req->payload_output);

From 2b7757bef345a3e61e4ef3520d31b8ceaa654939 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Fri, 20 May 2016 00:02:49 +0200
Subject: [PATCH 323/450] lib: Implemented net_set_send_buffer_size() and
 net_set_recv_buffer_size().

These functions allow manipulating the kernel socket buffer sizes for a socket file descriptor.
---
 src/lib/net.c | 24 ++++++++++++++++++++++++
 src/lib/net.h |  4 ++++
 2 files changed, 28 insertions(+)

diff --git a/src/lib/net.c b/src/lib/net.c
index 5223a544198..7450c1597a8 100644
--- a/src/lib/net.c
+++ b/src/lib/net.c
@@ -374,6 +374,30 @@ int net_set_cork(int fd ATTR_UNUSED, bool cork ATTR_UNUSED)
 #endif
 }
 
+int net_set_send_buffer_size(int fd, size_t size)
+{
+	int opt;
+
+	if (size > INT_MAX) {
+		errno = EINVAL;
+		return -1;
+	}
+	opt = (int)size;
+	return setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &opt, sizeof(opt));
+}
+
+int net_set_recv_buffer_size(int fd, size_t size)
+{
+	int opt;
+
+	if (size > INT_MAX) {
+		errno = EINVAL;
+		return -1;
+	}
+	opt = (int)size;
+	return setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &opt, sizeof(opt));
+}
+
 void net_get_ip_any4(struct ip_addr *ip)
 {
 	ip->family = AF_INET;
diff --git a/src/lib/net.h b/src/lib/net.h
index 59d89a6ad76..0a037705c91 100644
--- a/src/lib/net.h
+++ b/src/lib/net.h
@@ -93,6 +93,10 @@ void net_set_nonblock(int fd, bool nonblock);
    Returns 0 if ok, -1 if failed. */
 int net_set_cork(int fd, bool cork) ATTR_NOWARN_UNUSED_RESULT;
 
+/* Set socket kernel buffer sizes */
+int net_set_send_buffer_size(int fd, size_t size);
+int net_set_recv_buffer_size(int fd, size_t size);
+
 /* Set IP to contain INADDR_ANY for IPv4 or IPv6. The IPv6 any address may
    include IPv4 depending on the system (Linux yes, BSD no). */
 void net_get_ip_any4(struct ip_addr *ip);

From fff17849305e66e9310fb10175559ce50ae6ba00 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Fri, 20 May 2016 00:04:12 +0200
Subject: [PATCH 324/450] lib-http: client: Added settings to configure the
 connection's socket kernel buffer sizes.

This is mainly useful for use in the lib-http test suite.
---
 src/lib-http/http-client-connection.c | 13 +++++++++++++
 src/lib-http/http-client.c            |  2 ++
 src/lib-http/http-client.h            |  7 +++++++
 3 files changed, 22 insertions(+)

diff --git a/src/lib-http/http-client-connection.c b/src/lib-http/http-client-connection.c
index c25c903580e..84b8cc68a60 100644
--- a/src/lib-http/http-client-connection.c
+++ b/src/lib-http/http-client-connection.c
@@ -1184,6 +1184,7 @@ http_client_connection_connected(struct connection *_conn, bool success)
 {
 	struct http_client_connection *conn =
 		(struct http_client_connection *)_conn;
+	const struct http_client_settings *set = &conn->client->set;
 	const char *error;
 
 	if (!success) {
@@ -1192,6 +1193,18 @@ http_client_connection_connected(struct connection *_conn, bool success)
 	} else {
 		conn->connected_timestamp = ioloop_timeval;
 		http_client_connection_debug(conn, "Connected");
+
+		if (set->socket_send_buffer_size > 0) {
+			if (net_set_send_buffer_size(_conn->fd_out,
+				set->socket_send_buffer_size) < 0)
+				i_error("net_set_send_buffer_size() failed: %m");
+		}
+		if (set->socket_recv_buffer_size > 0) {
+			if (net_set_recv_buffer_size(_conn->fd_in,
+				set->socket_recv_buffer_size) < 0)
+				i_error("net_set_recv_buffer_size() failed: %m");
+		}
+
 		if (http_client_peer_addr_is_https(&conn->peer->addr)) {
 			if (http_client_connection_ssl_init(conn, &error) < 0) {
 				http_client_peer_connection_failure(conn->peer, error);
diff --git a/src/lib-http/http-client.c b/src/lib-http/http-client.c
index 7b034428c1b..75c38a51da6 100644
--- a/src/lib-http/http-client.c
+++ b/src/lib-http/http-client.c
@@ -149,6 +149,8 @@ struct http_client *http_client_init(const struct http_client_settings *set)
 	client->set.connect_timeout_msecs = set->connect_timeout_msecs;
 	client->set.soft_connect_timeout_msecs = set->soft_connect_timeout_msecs;
 	client->set.max_auto_retry_delay = set->max_auto_retry_delay;
+	client->set.socket_send_buffer_size = set->socket_send_buffer_size;
+	client->set.socket_recv_buffer_size = set->socket_recv_buffer_size;
 	client->set.debug = set->debug;
 
 	i_array_init(&client->delayed_failing_requests, 1);
diff --git a/src/lib-http/http-client.h b/src/lib-http/http-client.h
index fba5d1e0363..d482bb1399d 100644
--- a/src/lib-http/http-client.h
+++ b/src/lib-http/http-client.h
@@ -126,6 +126,13 @@ struct http_client_settings {
 	   is not automatically retried and the response is returned */
 	unsigned int max_auto_retry_delay;
 
+	/* the kernel send/receive buffer sizes used for the connection sockets.
+	   Configuring this is mainly useful for the test suite. The kernel
+	   defaults are used when these settings are 0. */
+	size_t socket_send_buffer_size;
+	size_t socket_recv_buffer_size;
+
+	/* enable logging debug messages */
 	bool debug;
 };
 

From 48e1753a801fc4e39b90091db15fe05accf13e68 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Sun, 22 May 2016 12:48:37 +0200
Subject: [PATCH 325/450] lib-http: client: Prevent useless and unexpected
 request callbacks during http_client_deinit().

Requests are now destroyed before queues, hosts, peers and connections.
As a side-effect, requests are now removed from the client request list at http_client_request_destroy(), so that requests with lingering references will no longer make http_client_wait() hang.
---
 src/lib-http/http-client-private.h |  1 +
 src/lib-http/http-client-request.c | 42 +++++++++++++++++++++++-------
 src/lib-http/http-client.c         | 24 ++++++++---------
 3 files changed, 46 insertions(+), 21 deletions(-)

diff --git a/src/lib-http/http-client-private.h b/src/lib-http/http-client-private.h
index 9f8e81eecbd..5005e10a33b 100644
--- a/src/lib-http/http-client-private.h
+++ b/src/lib-http/http-client-private.h
@@ -119,6 +119,7 @@ struct http_client_request {
 	unsigned int payload_wait:1;
 	unsigned int urgent:1;
 	unsigned int submitted:1;
+	unsigned int listed:1;
 	unsigned int connect_tunnel:1;
 	unsigned int connect_direct:1;
 	unsigned int ssl_tunnel:1;
diff --git a/src/lib-http/http-client-request.c b/src/lib-http/http-client-request.c
index 0c1a7013639..18222d89666 100644
--- a/src/lib-http/http-client-request.c
+++ b/src/lib-http/http-client-request.c
@@ -150,6 +150,32 @@ http_client_request_connect_ip(struct http_client *client,
 	return req;
 }
 
+static void
+http_client_request_add(struct http_client_request *req)
+{
+	struct http_client *client = req->client;
+
+	DLLIST_PREPEND(&client->requests_list, req);
+	client->requests_count++;
+	req->listed = TRUE;
+}
+
+static void
+http_client_request_remove(struct http_client_request *req)
+{
+	struct http_client *client = req->client;
+
+	if (req->listed) {
+		/* only decrease pending request counter if this request was submitted */
+		DLLIST_REMOVE(&client->requests_list, req);
+		client->requests_count--;
+	}
+	req->listed = FALSE;
+
+	if (client->requests_count == 0 && client->ioloop != NULL)
+		io_loop_stop(client->ioloop);
+}
+
 void http_client_request_ref(struct http_client_request *req)
 {
 	i_assert(req->refcount > 0);
@@ -182,11 +208,7 @@ bool http_client_request_unref(struct http_client_request **_req)
 		req->destroy_callback = NULL;
 	}
 
-	/* only decrease pending request counter if this request was submitted */
-	if (req->submitted) {
-		DLLIST_REMOVE(&client->requests_list, req);
-		client->requests_count--;
-	}
+	http_client_request_remove(req);
 
 	if (client->requests_count == 0 && client->ioloop != NULL)
 		io_loop_stop(client->ioloop);
@@ -213,6 +235,10 @@ void http_client_request_destroy(struct http_client_request **_req)
 	http_client_request_debug(req, "Destroy (requests left=%d)",
 		client->requests_count);
 
+	if (req->state < HTTP_REQUEST_STATE_FINISHED)
+		req->state = HTTP_REQUEST_STATE_ABORTED;
+	req->callback = NULL;
+
 	if (req->queue != NULL)
 		http_client_queue_drop_request(req->queue, req);
 
@@ -222,6 +248,7 @@ void http_client_request_destroy(struct http_client_request **_req)
 		req->destroy_callback = NULL;
 		callback(req->destroy_context);
 	}
+	http_client_request_remove(req);
 	http_client_request_unref(&req);
 }
 
@@ -562,16 +589,13 @@ static void http_client_request_do_submit(struct http_client_request *req)
 
 void http_client_request_submit(struct http_client_request *req)
 {
-	struct http_client *client = req->client;
-
 	req->submit_time = ioloop_timeval;
 
 	http_client_request_do_submit(req);
 	http_client_request_debug(req, "Submitted");
 
 	req->submitted = TRUE;
-	DLLIST_PREPEND(&client->requests_list, req);
-	client->requests_count++;
+	http_client_request_add(req);
 }
 
 void
diff --git a/src/lib-http/http-client.c b/src/lib-http/http-client.c
index 75c38a51da6..321b165ffe7 100644
--- a/src/lib-http/http-client.c
+++ b/src/lib-http/http-client.c
@@ -167,24 +167,20 @@ struct http_client *http_client_init(const struct http_client_settings *set)
 void http_client_deinit(struct http_client **_client)
 {
 	struct http_client *client = *_client;
-	struct http_client_request *req, *const *req_idx;
+	struct http_client_request *req;
 	struct http_client_host *host;
 	struct http_client_peer *peer;
 
 	*_client = NULL;
 
-	/* drop delayed failing requests */
-	while (array_count(&client->delayed_failing_requests) > 0) {
-		req_idx = array_idx(&client->delayed_failing_requests, 0);
-		req = *req_idx;
-
-		i_assert(req->refcount == 1);
-		http_client_request_error_delayed(&req);
+	/* destroy requests without calling callbacks */
+	req = client->requests_list;
+	while (req != NULL) {
+		struct http_client_request *next_req = req->next;
+		http_client_request_destroy(&req);
+		req = next_req;
 	}
-	array_free(&client->delayed_failing_requests);
-
-	if (client->to_failing_requests != NULL)
-		timeout_remove(&client->to_failing_requests);
+	i_assert(client->requests_count == 0);
 
 	/* free peers */
 	while (client->peers_list != NULL) {
@@ -200,6 +196,10 @@ void http_client_deinit(struct http_client **_client)
 	}
 	hash_table_destroy(&client->hosts);
 
+	array_free(&client->delayed_failing_requests);
+	if (client->to_failing_requests != NULL)
+		timeout_remove(&client->to_failing_requests);
+
 	connection_list_deinit(&client->conn_list);
 
 	if (client->ssl_ctx != NULL)

From 07b6d2bd9b8af054416b90cd0b816233e18364e3 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Mon, 23 May 2016 02:38:49 +0200
Subject: [PATCH 326/450] lib-http: client: Reworked connection close handling.

Now, the peer is immediately notified of the lost connection.
Before, this step was only taken when the connection was fully dereferenced.
To prevent recursive notifications between peer and connection, handling the loss of a connection is deferred to the request handler.
When a peer is freed, any associated lingering connections have conn->peer set to NULL.
---
 src/lib-http/http-client-connection.c | 65 +++++++++++++++++++--------
 src/lib-http/http-client-peer.c       | 11 +----
 src/lib-http/http-client-private.h    |  4 ++
 3 files changed, 52 insertions(+), 28 deletions(-)

diff --git a/src/lib-http/http-client-connection.c b/src/lib-http/http-client-connection.c
index 84b8cc68a60..c8f3565001f 100644
--- a/src/lib-http/http-client-connection.c
+++ b/src/lib-http/http-client-connection.c
@@ -46,8 +46,6 @@ http_client_connection_debug(struct http_client_connection *conn,
 
 static void http_client_connection_ready(struct http_client_connection *conn);
 static void http_client_connection_input(struct connection *_conn);
-static void
-http_client_connection_disconnect(struct http_client_connection *conn);
 
 unsigned int
 http_client_connection_count_pending(struct http_client_connection *conn)
@@ -561,8 +559,10 @@ static void http_client_payload_destroyed(struct http_client_request *req)
 	   somewhere from the API user's code, which we can't really know what
 	   state it is in). this call also triggers sending a new request if
 	   necessary. */
-	conn->to_input =
-		timeout_add_short(0, http_client_payload_destroyed_timeout, conn);
+	if (!conn->disconnected) {
+		conn->to_input = timeout_add_short
+			(0, http_client_payload_destroyed_timeout, conn);
+	}
 
 	/* room for new requests */
 	if (http_client_connection_is_ready(conn))
@@ -609,7 +609,8 @@ http_client_connection_return_response(
 	http_client_connection_ref(conn);
 	retrying = !http_client_request_callback(req, response);
 	tmp_conn = conn;
-	if (!http_client_connection_unref(&tmp_conn)) {
+	if (!http_client_connection_unref(&tmp_conn) ||
+		conn->disconnected) {
 		/* the callback managed to get this connection destroyed */
 		if (!retrying)
 			http_client_request_finish(req);
@@ -1414,6 +1415,16 @@ void http_client_connection_ref(struct http_client_connection *conn)
 static void
 http_client_connection_disconnect(struct http_client_connection *conn)
 {
+	struct http_client_peer *peer = conn->peer;
+	ARRAY_TYPE(http_client_connection) *conn_arr;
+	struct http_client_connection *const *conn_idx;
+
+	if (conn->disconnected)
+		return;
+	conn->disconnected = TRUE;
+
+	http_client_connection_debug(conn, "Connection disconnect");
+
 	conn->closing = TRUE;
 	conn->connected = FALSE;
 
@@ -1447,14 +1458,23 @@ http_client_connection_disconnect(struct http_client_connection *conn)
 		timeout_remove(&conn->to_idle);
 	if (conn->to_response != NULL)
 		timeout_remove(&conn->to_response);
+
+	/* remove this connection from the list */
+	conn_arr = &peer->conns;
+	array_foreach(conn_arr, conn_idx) {
+		if (*conn_idx == conn) {
+			array_delete(conn_arr, array_foreach_idx(conn_arr, conn_idx), 1);
+			break;
+		}
+	}
+
+	if (conn->connect_succeeded)
+		http_client_peer_connection_lost(peer);
 }
 
 bool http_client_connection_unref(struct http_client_connection **_conn)
 {
 	struct http_client_connection *conn = *_conn;
-	struct http_client_connection *const *conn_idx;
-	ARRAY_TYPE(http_client_connection) *conn_arr;
-	struct http_client_peer *peer = conn->peer;
 
 	i_assert(conn->refcount > 0);
 
@@ -1467,6 +1487,13 @@ bool http_client_connection_unref(struct http_client_connection **_conn)
 
 	http_client_connection_disconnect(conn);
 
+	i_assert(conn->io_req_payload == NULL);
+	i_assert(conn->to_requests == NULL);
+	i_assert(conn->to_connect == NULL);
+	i_assert(conn->to_input == NULL);
+	i_assert(conn->to_idle == NULL);
+	i_assert(conn->to_response == NULL);
+
 	if (array_is_created(&conn->request_wait_list))
 		array_free(&conn->request_wait_list);
 
@@ -1475,17 +1502,6 @@ bool http_client_connection_unref(struct http_client_connection **_conn)
 	if (conn->connect_initialized)
 		connection_deinit(&conn->conn);
 	
-	/* remove this connection from the list */
-	conn_arr = &conn->peer->conns;
-	array_foreach(conn_arr, conn_idx) {
-		if (*conn_idx == conn) {
-			array_delete(conn_arr, array_foreach_idx(conn_arr, conn_idx), 1);
-			break;
-		}
-	}
-
-	if (conn->connect_succeeded)
-		http_client_peer_connection_lost(peer);
 	i_free(conn);
 	return FALSE;
 }
@@ -1501,6 +1517,17 @@ void http_client_connection_close(struct http_client_connection **_conn)
 	http_client_connection_unref(_conn);
 }
 
+void http_client_connection_peer_closed(struct http_client_connection **_conn)
+{
+	struct http_client_connection *conn = *_conn;
+
+	http_client_connection_debug(conn, "Peer closed");
+	http_client_connection_disconnect(conn);
+
+	if (http_client_connection_unref(_conn))
+		conn->peer = NULL;
+}
+
 void http_client_connection_switch_ioloop(struct http_client_connection *conn)
 {
 	if (conn->io_req_payload != NULL)
diff --git a/src/lib-http/http-client-peer.c b/src/lib-http/http-client-peer.c
index 1f67899bf44..1876603ed55 100644
--- a/src/lib-http/http-client-peer.c
+++ b/src/lib-http/http-client-peer.c
@@ -534,7 +534,7 @@ http_client_peer_disconnect(struct http_client_peer *peer)
 	t_array_init(&conns, array_count(&peer->conns));
 	array_copy(&conns.arr, 0, &peer->conns.arr, 0, array_count(&peer->conns));
 	array_foreach_modifiable(&conns, conn) {
-		http_client_connection_unref(conn);
+		http_client_connection_peer_closed(conn);
 	}
 	i_assert(array_count(&peer->conns) == 0);
 
@@ -746,15 +746,8 @@ void http_client_peer_connection_lost(struct http_client_peer *peer)
 		return;
 	}
 
-	/* check if peer is still relevant */
-	if (array_count(&peer->conns) == 0 &&
-		http_client_peer_requests_pending(peer, &num_urgent) == 0) {
-		http_client_peer_close(&peer);
-		return;
-	}
-
 	/* if there are pending requests for this peer, create a new connection
-	   for them. */
+	   for them. if not, this peer will wind itself down. */
 	http_client_peer_trigger_request_handler(peer);
 }
 
diff --git a/src/lib-http/http-client-private.h b/src/lib-http/http-client-private.h
index 5005e10a33b..fd28af20e62 100644
--- a/src/lib-http/http-client-private.h
+++ b/src/lib-http/http-client-private.h
@@ -159,6 +159,7 @@ struct http_client_connection {
 	unsigned int connect_initialized:1; /* connection was initialized */
 	unsigned int connect_succeeded:1;
 	unsigned int closing:1;
+	unsigned int disconnected:1;
 	unsigned int close_indicated:1;
 	unsigned int output_locked:1;       /* output is locked; no pipelining */
 	unsigned int output_broken:1;       /* output is broken; no more requests */
@@ -312,6 +313,9 @@ void http_client_connection_ref(struct http_client_connection *conn);
 /* Returns FALSE if unrefing destroyed the connection entirely */
 bool http_client_connection_unref(struct http_client_connection **_conn);
 void http_client_connection_close(struct http_client_connection **_conn);
+
+void http_client_connection_peer_closed(struct http_client_connection **_conn);
+
 int http_client_connection_output(struct http_client_connection *conn);
 void http_client_connection_start_request_timeout(
 	struct http_client_connection *conn);

From 8d558a5763725741c77682a4229014222252b4e0 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Wed, 25 May 2016 23:41:47 +0200
Subject: [PATCH 327/450] lib-http: client: Fixed bug in handling of lost
 connections while returning from another ioloop.

At one instance the http_client_connection_is_ready() function could have destroyed the connection while the caller still depended on it.
Renamed the http_client_connection_is_ready() function to http_client_connection_check_ready().
This now returns -1 when the connection got destroyed. Before it returned a bool that just indicated whether the connection was ready or not.
So, there is no need anymore to preserve a connection reference while calling this function.
---
 src/lib-http/http-client-connection.c | 29 ++++++++++++++-------------
 src/lib-http/http-client-peer.c       | 12 +++++------
 src/lib-http/http-client-private.h    |  2 +-
 3 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/src/lib-http/http-client-connection.c b/src/lib-http/http-client-connection.c
index c8f3565001f..880e89a101e 100644
--- a/src/lib-http/http-client-connection.c
+++ b/src/lib-http/http-client-connection.c
@@ -235,7 +235,7 @@ http_client_connection_abort_temp_error(struct http_client_connection **_conn,
 	http_client_connection_close(_conn);
 }
 
-bool http_client_connection_is_ready(struct http_client_connection *conn)
+int http_client_connection_check_ready(struct http_client_connection *conn)
 {
 	int ret;
 
@@ -245,14 +245,14 @@ bool http_client_connection_is_ready(struct http_client_connection *conn)
 		   this way, but theoretically we could, although that would add
 		   quite a bit of complexity.
 		 */
-		return FALSE;
+		return 0;
 	}
 
 	if (!conn->connected || conn->output_locked || conn->output_broken ||
 		conn->close_indicated || conn->tunneling ||
 		http_client_connection_count_pending(conn) >=
 			conn->client->set.max_pipelined_requests)
-		return FALSE;
+		return 0;
 
 	if (conn->last_ioloop != NULL && conn->last_ioloop != current_ioloop) {
 		conn->last_ioloop = current_ioloop;
@@ -270,10 +270,10 @@ bool http_client_connection_is_ready(struct http_client_connection *conn)
 						stream_errno != 0 ?
 						i_stream_get_error(conn->conn.input) :
 						"EOF"));
-			return FALSE;
+			return -1;
 		}
 	}
-	return TRUE;
+	return 1;
 }
 
 static void
@@ -408,10 +408,14 @@ int http_client_connection_next_request(struct http_client_connection *conn)
 	struct http_client_request *req = NULL;
 	const char *error;
 	bool pipelined;
+	int ret;
 
-	if (!http_client_connection_is_ready(conn)) {
-		http_client_connection_debug(conn, "Not ready for next request");
-		return 0;
+	if ((ret=http_client_connection_check_ready(conn)) <= 0) {
+		if (ret == 0) {
+			http_client_connection_debug(conn,
+				"Not ready for next request");
+		}
+		return ret;
 	}
 
 	/* claim request, but no urgent request can be second in line */
@@ -552,7 +556,6 @@ static void http_client_payload_destroyed(struct http_client_request *req)
 	}
 
 	conn->incoming_payload = NULL;
-	http_client_connection_ref(conn);
 
 	/* input stream may have pending input. make sure input handler
 	   gets called (but don't do it directly, since we get get here
@@ -565,10 +568,8 @@ static void http_client_payload_destroyed(struct http_client_request *req)
 	}
 
 	/* room for new requests */
-	if (http_client_connection_is_ready(conn))
+	if (http_client_connection_check_ready(conn) > 0)
 		http_client_peer_trigger_request_handler(conn->peer);
-
-	http_client_connection_unref(&conn);
 }
 
 static bool
@@ -966,7 +967,7 @@ static void http_client_connection_input(struct connection *_conn)
 		conn->peer->allows_pipelining = TRUE;
 
 		/* room for new requests */
-		if (http_client_connection_is_ready(conn))
+		if (http_client_connection_check_ready(conn) > 0)
 			http_client_peer_trigger_request_handler(conn->peer);
 	}
 }
@@ -1025,7 +1026,7 @@ int http_client_connection_output(struct http_client_connection *conn)
 			}
 			if (!conn->output_locked) {
 				/* room for new requests */
-				if (http_client_connection_is_ready(conn))
+				if (http_client_connection_check_ready(conn) > 0)
 					http_client_peer_trigger_request_handler(conn->peer);
 			}
 		}
diff --git a/src/lib-http/http-client-peer.c b/src/lib-http/http-client-peer.c
index 1876603ed55..d963e03f2b0 100644
--- a/src/lib-http/http-client-peer.c
+++ b/src/lib-http/http-client-peer.c
@@ -268,9 +268,12 @@ http_client_peer_handle_requests_real(struct http_client_peer *peer)
 		/* gather connection statistics */
 		array_foreach(&peer->conns, conn_idx) {
 			struct http_client_connection *conn = *conn_idx;
+			int ret;
 
-			http_client_connection_ref(conn);
-			if (http_client_connection_is_ready(conn)) {
+			if ((ret=http_client_connection_check_ready(conn)) < 0) {
+				conn_lost = TRUE;
+				break;
+			} else if (ret > 0) {
 				struct _conn_available *conn_avail;
 				unsigned int insert_idx, pending_requests;
 
@@ -293,11 +296,6 @@ http_client_peer_handle_requests_real(struct http_client_peer *peer)
 				if (pending_requests == 0)
 					idle++;
 			}
-			if (!http_client_connection_unref(&conn)) {
-				conn_lost = TRUE;
-				break;
-			}
-			conn = *conn_idx;
 			/* count the number of connecting and closing connections */
 			if (conn->closing)
 				closing++;
diff --git a/src/lib-http/http-client-private.h b/src/lib-http/http-client-private.h
index fd28af20e62..f62630ff432 100644
--- a/src/lib-http/http-client-private.h
+++ b/src/lib-http/http-client-private.h
@@ -325,7 +325,7 @@ void http_client_connection_stop_request_timeout(
 	struct http_client_connection *conn);
 unsigned int
 http_client_connection_count_pending(struct http_client_connection *conn);
-bool http_client_connection_is_ready(struct http_client_connection *conn);
+int http_client_connection_check_ready(struct http_client_connection *conn);
 bool http_client_connection_is_idle(struct http_client_connection *conn);
 int http_client_connection_next_request(struct http_client_connection *conn);
 void http_client_connection_check_idle(struct http_client_connection *conn);

From 4c4ec1720ed08661c1963cd4beb4f217c6bbcea5 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Mon, 23 May 2016 03:16:07 +0200
Subject: [PATCH 328/450] lib-http: test-http-client: Callback is not called
 for explicitly aborted requests.

Test request context was not freed in that case.
---
 src/lib-http/test-http-client.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lib-http/test-http-client.c b/src/lib-http/test-http-client.c
index 883cf52ddbb..3b3a26c156c 100644
--- a/src/lib-http/test-http-client.c
+++ b/src/lib-http/test-http-client.c
@@ -182,6 +182,7 @@ static void run_tests(struct http_client *http_client)
 	http_client_request_set_ssl(http_req, TRUE);
 	http_client_request_submit(http_req);
 	http_client_request_abort(&http_req);
+	i_free(test_req);
 
 	test_req = i_new(struct http_test_request, 1);
 	http_req = http_client_request(http_client,

From 5ad09b076d224d517d77020d3b821e5979881238 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Sun, 22 May 2016 19:16:42 +0200
Subject: [PATCH 329/450] lib-http: test-http-payload: Added -D option to
 easily enable debug mode.

---
 src/lib-http/test-http-payload.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/lib-http/test-http-payload.c b/src/lib-http/test-http-payload.c
index 6e85be8b04e..3dc4b865c48 100644
--- a/src/lib-http/test-http-payload.c
+++ b/src/lib-http/test-http-payload.c
@@ -1381,8 +1381,20 @@ static void (*test_functions[])(void) = {
  * Main
  */
 
-int main(void)
+int main(int argc, char *argv[])
 {
+	int c;
+
+  while ((c = getopt(argc, argv, "D")) > 0) {
+		switch (c) {
+		case 'D':
+			debug = TRUE;
+			break;
+		default:
+			i_fatal("Usage: %s [-D]", argv[0]);
+		}
+  }
+
 	/* listen on localhost */
 	memset(&bind_ip, 0, sizeof(bind_ip));
 	bind_ip.family = AF_INET;

From 9ae8eaf77cd188f3e29fbca045fd8ec6cdfe6ffe Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Mon, 23 May 2016 21:57:18 +0200
Subject: [PATCH 330/450] lib-http: test-http-payload: Added cleanup code to
 make sure no child processes get orphaned at failures.

Also handles SIGSEGV and SIGABRT.
---
 src/lib-http/test-http-payload.c | 50 +++++++++++++++++++++++++++-----
 1 file changed, 43 insertions(+), 7 deletions(-)

diff --git a/src/lib-http/test-http-payload.c b/src/lib-http/test-http-payload.c
index 3dc4b865c48..2497414a4ac 100644
--- a/src/lib-http/test-http-payload.c
+++ b/src/lib-http/test-http-payload.c
@@ -35,6 +35,7 @@ static unsigned int test_max_pending = 200;
 static struct ip_addr bind_ip;
 static in_port_t bind_port = 0;
 static int fd_listen = -1;
+static pid_t server_pid = (pid_t)-1;
 
 /*
  * Test files
@@ -1101,20 +1102,28 @@ static void test_open_server_fd(void)
 	}
 }
 
+static void test_server_kill(void)
+{
+	if (server_pid != (pid_t)-1) {
+		(void)kill(server_pid, SIGKILL);
+		(void)waitpid(server_pid, NULL, 0);
+	}
+	server_pid = (pid_t)-1;
+}
+
 static void test_run_client_server(
 	const struct http_client_settings *client_set,
 	const struct http_server_settings *server_set,
 	void (*client_init)(const struct http_client_settings *client_set))
 {
 	struct ioloop *ioloop;
-	pid_t pid;
-	int status;
 
 	test_open_server_fd();
 
-	if ((pid = fork()) == (pid_t)-1)
+	if ((server_pid = fork()) == (pid_t)-1)
 		i_fatal("fork() failed: %m");
-	if (pid == 0) {
+	if (server_pid == 0) {
+		server_pid = (pid_t)-1;
 		hostpid_init();
 		if (debug)
 			i_debug("server: PID=%s", my_pid);
@@ -1135,9 +1144,7 @@ static void test_run_client_server(
 		io_loop_run(ioloop);
 		test_client_deinit();
 		io_loop_destroy(&ioloop);
-
-		(void)kill(pid, SIGKILL);
-		(void)waitpid(pid, &status, 0);
+		test_server_kill();
 	}
 }
 
@@ -1381,10 +1388,39 @@ static void (*test_functions[])(void) = {
  * Main
  */
 
+volatile sig_atomic_t terminating = 0;
+
+static void
+test_signal_handler(int signo)
+{
+	if (terminating)
+		raise(signo);
+	terminating = 1;
+
+	/* make sure we don't leave any pesky children alive */
+	test_server_kill();
+
+	(void)signal(signo, SIG_DFL);
+	raise(signo);
+}
+
+static void test_atexit(void)
+{
+	test_server_kill();
+}
+
 int main(int argc, char *argv[])
 {
 	int c;
 
+	atexit(test_atexit);
+	(void)signal(SIGCHLD, SIG_IGN);
+	(void)signal(SIGTERM, test_signal_handler);
+	(void)signal(SIGQUIT, test_signal_handler);
+	(void)signal(SIGINT, test_signal_handler);
+	(void)signal(SIGSEGV, test_signal_handler);
+	(void)signal(SIGABRT, test_signal_handler);
+
   while ((c = getopt(argc, argv, "D")) > 0) {
 		switch (c) {
 		case 'D':

From ddb65fbdc2478d83caab5e7a05fae411500a0e41 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Tue, 24 May 2016 03:11:07 +0200
Subject: [PATCH 331/450] lib-http: test-http-payload: Added tests for the use
 of nested ioloops.

---
 src/lib-http/test-http-payload.c | 166 ++++++++++++++++++++++++++-----
 1 file changed, 142 insertions(+), 24 deletions(-)

diff --git a/src/lib-http/test-http-payload.c b/src/lib-http/test-http-payload.c
index 2497414a4ac..047ff993db7 100644
--- a/src/lib-http/test-http-payload.c
+++ b/src/lib-http/test-http-payload.c
@@ -31,11 +31,15 @@ static bool request_100_continue = FALSE;
 static size_t read_server_partial = 0;
 static size_t read_client_partial = 0;
 static unsigned int test_max_pending = 200;
+static unsigned int client_ioloop_nesting = 0;
 
 static struct ip_addr bind_ip;
 static in_port_t bind_port = 0;
 static int fd_listen = -1;
 static pid_t server_pid = (pid_t)-1;
+static struct ioloop *ioloop_nested = NULL;
+static unsigned ioloop_nested_first = 0;
+static unsigned ioloop_nested_last = 0;
 
 /*
  * Test files
@@ -583,6 +587,9 @@ static void test_server_deinit(void)
  */
 
 struct test_client_request {
+	struct test_client_request *prev, *next;
+	struct http_client_request *hreq;
+
 	struct io *io;
 	struct istream *payload;
 	struct istream *file;
@@ -590,8 +597,20 @@ struct test_client_request {
 };
 
 static struct http_client *http_client;
+static struct test_client_request *client_requests;
 static unsigned int client_files_first, client_files_last;
 
+static struct test_client_request *
+test_client_request_new(void)
+{
+	struct test_client_request *tcreq;
+
+	tcreq = i_new(struct test_client_request, 1);
+	DLLIST_PREPEND(&client_requests, tcreq);
+
+	return tcreq;
+}
+
 static void
 test_client_request_destroy(void *context)
 {
@@ -604,25 +623,41 @@ test_client_request_destroy(void *context)
 		i_stream_unref(&tcreq->payload);
 	if (tcreq->file != NULL)
 		i_stream_unref(&tcreq->file);
+
+	DLLIST_REMOVE(&client_requests, tcreq);
 	i_free(tcreq);
 }
 
+static void
+test_client_requests_switch_ioloop(void)
+{
+	struct test_client_request *tcreq;
+
+	for (tcreq = client_requests; tcreq != NULL;
+		tcreq = tcreq->next) {
+		if (tcreq->io != NULL)
+			tcreq->io = io_loop_move_io(&tcreq->io);
+		if (tcreq->payload != NULL)
+			i_stream_switch_ioloop(tcreq->payload);
+	}
+}
+
 /* download */
 
 static void test_client_download_continue(void);
 
 static void
-test_client_download_finished(struct test_client_request *tcreq)
+test_client_download_finished(unsigned int files_idx)
 {
 	const char **paths;
 	unsigned int count;
 
 	paths = array_get_modifiable(&files, &count);
-	i_assert(tcreq->files_idx < count);
+	i_assert(files_idx < count);
 	i_assert(client_files_first < count);
-	i_assert(paths[tcreq->files_idx] != NULL);
+	i_assert(paths[files_idx] != NULL);
 
-	paths[tcreq->files_idx] = NULL;
+	paths[files_idx] = NULL;
 	test_client_download_continue();
 }
 
@@ -632,6 +667,7 @@ test_client_download_payload_input(struct test_client_request *tcreq)
 	struct istream *payload = tcreq->payload;
 	const unsigned char *pdata, *fdata;
 	size_t psize, fsize, pleft;
+	unsigned int files_idx = tcreq->files_idx;
 	off_t ret;
 
 	/* read payload */
@@ -689,13 +725,13 @@ test_client_download_payload_input(struct test_client_request *tcreq)
 				tcreq->files_idx);
 		}
 
-		/* finished */
-		test_client_download_finished(tcreq);
-
 		/* dereference payload stream; finishes the request */
 		tcreq->payload = NULL;
 		io_remove(&tcreq->io); /* holds a reference too */
 		i_stream_unref(&payload);
+
+		/* finished */
+		test_client_download_finished(files_idx);
 	}
 }
 
@@ -741,7 +777,7 @@ test_client_download_response(const struct http_response *resp,
 				path, resp->status, resp->reason);
 		}
 		i_stream_unref(&fstream);
-		test_client_download_finished(tcreq);
+		test_client_download_finished(tcreq->files_idx);
 		return;
 	}
 
@@ -752,7 +788,7 @@ test_client_download_response(const struct http_response *resp,
 				path, tcreq->files_idx);
 		}
 		i_stream_unref(&fstream);
-		test_client_download_finished(tcreq);
+		test_client_download_finished(tcreq->files_idx);
 		return;
 	}
 
@@ -806,7 +842,7 @@ static void test_client_download_continue(void)
 		client_files_last++) {
 		const char *path = paths[client_files_last];
 
-		tcreq = i_new(struct test_client_request, 1);
+		tcreq = test_client_request_new();
 		tcreq->files_idx = client_files_last;
 
 		if (debug) {
@@ -814,7 +850,7 @@ static void test_client_download_continue(void)
 				"retrieving %s [%u]",
 				path, tcreq->files_idx);
 		}
-		hreq = http_client_request(http_client,
+		hreq = tcreq->hreq = http_client_request(http_client,
 			"GET", net_ip2addr(&bind_ip),
 			t_strconcat("/download/", path, NULL),
 			test_client_download_response, tcreq);
@@ -841,17 +877,17 @@ test_client_download(const struct http_client_settings *client_set)
 static void test_client_echo_continue(void);
 
 static void
-test_client_echo_finished(struct test_client_request *tcreq)
+test_client_echo_finished(unsigned int files_idx)
 {
 	const char **paths;
 	unsigned int count;
 
 	paths = array_get_modifiable(&files, &count);
-	i_assert(tcreq->files_idx < count);
+	i_assert(files_idx < count);
 	i_assert(client_files_first < count);
-	i_assert(paths[tcreq->files_idx] != NULL);
+	i_assert(paths[files_idx] != NULL);
 
-	paths[tcreq->files_idx] = NULL;
+	paths[files_idx] = NULL;
 	test_client_echo_continue();
 }
 
@@ -861,6 +897,7 @@ test_client_echo_payload_input(struct test_client_request *tcreq)
 	struct istream *payload = tcreq->payload;
 	const unsigned char *pdata, *fdata;
 	size_t psize, fsize, pleft;
+	unsigned int files_idx = tcreq->files_idx;
 	off_t ret;
 
 	/* read payload */
@@ -918,13 +955,13 @@ test_client_echo_payload_input(struct test_client_request *tcreq)
 				tcreq->files_idx);
 		}
 
-		/* finished */
-		test_client_echo_finished(tcreq);
-
 		/* dereference payload stream; finishes the request */
 		tcreq->payload = NULL;
 		io_remove(&tcreq->io); /* holds a reference too */
 		i_stream_unref(&payload);
+
+		/* finished */
+		test_client_echo_finished(files_idx);
 	}
 }
 
@@ -982,7 +1019,7 @@ test_client_echo_response(const struct http_response *resp,
 				path, tcreq->files_idx);
 		}
 		i_stream_unref(&fstream);
-		test_client_echo_finished(tcreq);
+		test_client_echo_finished(tcreq->files_idx);
 		return;
 	}
 
@@ -1001,7 +1038,7 @@ static void test_client_echo_continue(void)
 	struct test_client_request *tcreq;
 	struct http_client_request *hreq;
 	const char **paths;
-	unsigned int count;
+	unsigned int count, first_submitted;
 
 	paths = array_get_modifiable(&files, &count);
 
@@ -1010,7 +1047,7 @@ static void test_client_echo_continue(void)
 
 	i_assert(client_files_first <= client_files_last);
 	for (; client_files_first < client_files_last &&
-		paths[client_files_first] == NULL; client_files_first++)
+		paths[client_files_first] == NULL; client_files_first++);
 
 	if (debug) {
 		i_debug("test client: echo: "
@@ -1020,7 +1057,9 @@ static void test_client_echo_continue(void)
 	if (debug && client_files_first < count) {
 		const char *path = paths[client_files_first];
 		i_debug("test client: echo: "
-			"next blocking: %s", (path == NULL ? "none" : path));
+			"next blocking: %s [%d]",
+			(path == NULL ? "none" : path),
+			client_files_first);
 	}
 
 	if (client_files_first >= count) {
@@ -1028,6 +1067,7 @@ static void test_client_echo_continue(void)
 		return;
 	}
 
+	first_submitted = client_files_last;
 	for (; client_files_last < count &&
 			(client_files_last - client_files_first) < test_max_pending;
 		client_files_last++) {
@@ -1051,10 +1091,10 @@ static void test_client_echo_continue(void)
 				path, client_files_last);
 		}
 
-		tcreq = i_new(struct test_client_request, 1);
+		tcreq = test_client_request_new();
 		tcreq->files_idx = client_files_last;
 
-		hreq = http_client_request(http_client,
+		hreq = tcreq->hreq = http_client_request(http_client,
 			"PUT", net_ip2addr(&bind_ip),
 			t_strconcat("/echo/", path, NULL),
 			test_client_echo_response, tcreq);
@@ -1067,6 +1107,60 @@ static void test_client_echo_continue(void)
 
 		i_stream_unref(&fstream);
 	}
+
+	/* run nested ioloop (if requested) if new requests cross a nesting
+	   boundary */
+	if (ioloop_nested != NULL) {
+		unsigned int i;
+
+		i_assert(ioloop_nested_first <= count);
+		i_assert(ioloop_nested_last <= count);
+		for (i = ioloop_nested_first; i < ioloop_nested_last; i++) {
+			if (paths[i] != NULL) {
+				if (debug) {
+					i_debug("test client: "
+						"not leaving ioloop [%u]", i);
+				}
+				break;
+			}
+		}
+
+		if (i == ioloop_nested_last)
+			io_loop_stop(ioloop_nested);
+	} else if (client_ioloop_nesting > 0 &&
+		((client_files_last / client_ioloop_nesting) !=
+			(first_submitted / client_ioloop_nesting)) ) {
+		struct ioloop *prev_ioloop = current_ioloop;
+
+		ioloop_nested_first = first_submitted;
+		ioloop_nested_last = first_submitted + client_ioloop_nesting;
+		if (ioloop_nested_last > client_files_last)
+			ioloop_nested_last = client_files_last;
+
+		if (debug) {
+			i_debug("test client: echo: entering ioloop for %u...%u",
+				ioloop_nested_first, ioloop_nested_last);
+		}
+
+		ioloop_nested = io_loop_create();
+		http_client_switch_ioloop(http_client);
+		test_client_requests_switch_ioloop();
+
+		io_loop_run(ioloop_nested);
+
+		io_loop_set_current(prev_ioloop);
+		http_client_switch_ioloop(http_client);
+		test_client_requests_switch_ioloop();
+		io_loop_set_current(ioloop_nested);
+		io_loop_destroy(&ioloop_nested);
+		ioloop_nested = NULL;
+
+		if (debug) {
+			i_debug("test client: echo: leaving ioloop for %u...%u",
+				ioloop_nested_first, ioloop_nested_last);
+		}
+		ioloop_nested_first = ioloop_nested_last = 0;
+	}
 }
 
 static void
@@ -1128,6 +1222,7 @@ static void test_run_client_server(
 		if (debug)
 			i_debug("server: PID=%s", my_pid);
 		/* child: server */
+		ioloop_nested = FALSE;
 		ioloop = io_loop_create();
 		test_server_init(server_set);
 		io_loop_run(ioloop);
@@ -1139,6 +1234,7 @@ static void test_run_client_server(
 			i_debug("client: PID=%s", my_pid);
 		i_close_fd(&fd_listen);
 		/* parent: client */
+		ioloop_nested = FALSE;
 		ioloop = io_loop_create();
 		client_init(client_set);
 		io_loop_run(ioloop);
@@ -1247,6 +1343,7 @@ static void test_download_server_nonblocking(void)
 	blocking = FALSE;
 	request_100_continue = FALSE;
 	read_server_partial = 0;
+	client_ioloop_nesting = 0;
 	test_run_sequential(test_client_download);
 	test_run_pipeline(test_client_download);
 	test_run_parallel(test_client_download);
@@ -1259,6 +1356,7 @@ static void test_download_server_blocking(void)
 	blocking = TRUE;
 	request_100_continue = FALSE;
 	read_server_partial = 0;
+	client_ioloop_nesting = 0;
 	test_run_sequential(test_client_download);
 	test_run_pipeline(test_client_download);
 	test_run_parallel(test_client_download);
@@ -1271,6 +1369,7 @@ static void test_echo_server_nonblocking(void)
 	blocking = FALSE;
 	request_100_continue = FALSE;
 	read_server_partial = 0;
+	client_ioloop_nesting = 0;
 	test_run_sequential(test_client_echo);
 	test_run_pipeline(test_client_echo);
 	test_run_parallel(test_client_echo);
@@ -1283,6 +1382,7 @@ static void test_echo_server_blocking(void)
 	blocking = TRUE;
 	request_100_continue = FALSE;
 	read_server_partial = 0;
+	client_ioloop_nesting = 0;
 	test_run_sequential(test_client_echo);
 	test_run_pipeline(test_client_echo);
 	test_run_parallel(test_client_echo);
@@ -1295,6 +1395,7 @@ static void test_echo_server_nonblocking_sync(void)
 	blocking = FALSE;
 	request_100_continue = TRUE;
 	read_server_partial = 0;
+	client_ioloop_nesting = 0;
 	test_run_sequential(test_client_echo);
 	test_run_pipeline(test_client_echo);
 	test_run_parallel(test_client_echo);
@@ -1307,6 +1408,7 @@ static void test_echo_server_blocking_sync(void)
 	blocking = TRUE;
 	request_100_continue = TRUE;
 	read_server_partial = 0;
+	client_ioloop_nesting = 0;
   test_run_sequential(test_client_echo);
 	test_run_pipeline(test_client_echo);
 	test_run_parallel(test_client_echo);
@@ -1319,6 +1421,7 @@ static void test_echo_server_nonblocking_partial(void)
 	blocking = FALSE;
 	request_100_continue = FALSE;
 	read_server_partial = 1024;
+	client_ioloop_nesting = 0;
 	test_run_sequential(test_client_echo);
 	test_run_pipeline(test_client_echo);
 	test_run_parallel(test_client_echo);
@@ -1337,6 +1440,7 @@ static void test_echo_server_blocking_partial(void)
 	blocking = TRUE;
 	request_100_continue = FALSE;
 	read_server_partial = 1024;
+	client_ioloop_nesting = 0;
 	test_run_sequential(test_client_echo);
 	test_run_pipeline(test_client_echo);
 	test_run_parallel(test_client_echo);
@@ -1356,6 +1460,7 @@ static void test_download_client_partial(void)
 	request_100_continue = FALSE;
 	read_server_partial = 0;
 	read_client_partial = 1024;
+	client_ioloop_nesting = 0;
 	test_run_sequential(test_client_download);
 	test_run_pipeline(test_client_download);
 	test_run_parallel(test_client_download);
@@ -1371,6 +1476,18 @@ static void test_download_client_partial(void)
 	test_end();
 }
 
+static void test_download_client_nested_ioloop(void)
+{
+	test_begin("http payload echo (client nested ioloop)");
+	blocking = FALSE;
+	request_100_continue = FALSE;
+	read_server_partial = 0;
+	read_client_partial = 0;
+	client_ioloop_nesting = 10;
+	test_run_parallel(test_client_echo);
+	test_end();
+}
+
 static void (*test_functions[])(void) = {
 	test_download_server_nonblocking,
 	test_download_server_blocking,
@@ -1381,6 +1498,7 @@ static void (*test_functions[])(void) = {
 	test_echo_server_nonblocking_partial,
 	test_echo_server_blocking_partial,
 	test_download_client_partial,
+	test_download_client_nested_ioloop,
 	NULL
 };
 

From 534195d811f76b41337b24a9c9cc9889dfa1a352 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Sun, 22 May 2016 10:38:18 +0200
Subject: [PATCH 332/450] lib-http: client: Created test program that triggers
 most possible error conditions.

---
 src/lib-http/Makefile.am               |   18 +
 src/lib-http/test-http-client-errors.c | 1797 ++++++++++++++++++++++++
 2 files changed, 1815 insertions(+)
 create mode 100644 src/lib-http/test-http-client-errors.c

diff --git a/src/lib-http/Makefile.am b/src/lib-http/Makefile.am
index 41d93816e35..73941e8f4ea 100644
--- a/src/lib-http/Makefile.am
+++ b/src/lib-http/Makefile.am
@@ -64,6 +64,7 @@ test_programs = \
 test_nocheck_programs = \
 	test-http-payload \
 	test-http-client \
+	test-http-client-errors \
 	test-http-server
 
 noinst_PROGRAMS = $(test_programs) $(test_nocheck_programs)
@@ -165,6 +166,23 @@ test_http_client_DEPENDENCIES = \
 	../lib-settings/libsettings.la \
 	$(test_deps)
 
+test_http_client_errors_SOURCES = test-http-client-errors.c
+test_http_client_errors_LDFLAGS = -export-dynamic
+test_http_client_errors_LDADD = \
+	libhttp.la \
+	../lib-dns/libdns.la \
+	../lib-ssl-iostream/libssl_iostream.la \
+	../lib-master/libmaster.la \
+	../lib-settings/libsettings.la \
+	$(test_libs)
+test_http_client_errors_DEPENDENCIES = \
+	libhttp.la \
+	../lib-dns/libdns.la \
+	../lib-ssl-iostream/libssl_iostream.la \
+	../lib-master/libmaster.la \
+	../lib-settings/libsettings.la \
+	$(test_deps)
+
 test_http_server_SOURCES = test-http-server.c
 test_http_server_LDFLAGS = -export-dynamic
 test_http_server_LDADD = \
diff --git a/src/lib-http/test-http-client-errors.c b/src/lib-http/test-http-client-errors.c
new file mode 100644
index 00000000000..36f88891c42
--- /dev/null
+++ b/src/lib-http/test-http-client-errors.c
@@ -0,0 +1,1797 @@
+/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "hostpid.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "ostream.h"
+#include "connection.h"
+#include "test-common.h"
+#include "http-url.h"
+#include "http-request.h"
+#include "http-client.h"
+
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <signal.h>
+#include <unistd.h>
+
+/*
+ * Types
+ */
+
+struct server_connection {
+	struct connection conn;
+
+	pool_t pool;
+};
+
+typedef void (*test_server_init_t)(unsigned int index);
+typedef void (*test_client_init_t)
+	(const struct http_client_settings *client_set);
+
+/*
+ * State
+ */
+
+/* common */
+static struct ip_addr bind_ip;
+static in_port_t *bind_ports = 0;
+static struct ioloop *ioloop;
+static bool debug = FALSE;
+
+/* server */
+static struct io *io_listen;
+static int fd_listen = -1;
+static pid_t *server_pids = NULL;
+static unsigned int server_pids_count = 0;
+static struct connection_list *server_conn_list;
+static size_t server_read_max = 0;
+static unsigned int server_index;
+static void (*test_server_input)(struct server_connection *conn);
+
+/* client */
+static struct http_client *http_client = NULL;
+
+/*
+ * Forward declarations
+ */
+
+/* server */
+static void test_server_run(unsigned int index);
+static void
+server_connection_deinit(struct server_connection **_conn);
+
+/* client */
+static void
+test_client_defaults(struct http_client_settings *http_set);
+static void test_client_deinit(void);
+
+/* test*/
+static void test_run_client_server(
+	const struct http_client_settings *client_set,
+	test_client_init_t client_test,
+	test_server_init_t server_test,
+	unsigned int server_tests_count)
+	ATTR_NULL(3);
+
+/*
+ * Host lookup failed
+ */
+
+/* client */
+
+struct _host_lookup_failed {
+	unsigned int count;
+};
+
+static void
+test_client_host_lookup_failed_response(
+	const struct http_response *resp,
+	struct _host_lookup_failed *ctx)
+{
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_HOST_LOOKUP_FAILED);
+	test_assert(resp->reason != NULL && *resp->reason != '\0');
+
+	if (--ctx->count == 0) {
+		i_free(ctx);
+		io_loop_stop(ioloop);
+	}
+}
+
+static void
+test_client_host_lookup_failed(const struct http_client_settings *client_set)
+{
+	struct http_client_request *hreq;
+	struct _host_lookup_failed *ctx;
+
+	ctx = i_new(struct _host_lookup_failed, 1);
+	ctx->count = 2;
+
+	http_client = http_client_init(client_set);
+
+	hreq = http_client_request(http_client,
+		"GET", "host.in-addr.arpa", "/host-lookup-failed.txt",
+		test_client_host_lookup_failed_response, ctx);
+	http_client_request_submit(hreq);
+
+	hreq = http_client_request(http_client,
+		"GET", "host.in-addr.arpa", "/host-lookup-failed2.txt",
+		test_client_host_lookup_failed_response, ctx);
+	http_client_request_submit(hreq);
+}
+
+/* test */
+
+static void test_host_lookup_failed(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+
+	test_begin("host lookup failed");
+	test_run_client_server(&http_client_set,
+		test_client_host_lookup_failed,
+		NULL, 0);
+	test_end();
+}
+
+/*
+ * Connection refused
+ */
+
+/* server */
+
+static void
+test_server_connection_refused(unsigned int index ATTR_UNUSED)
+{
+	i_close_fd(&fd_listen);
+}
+
+/* client */
+
+struct _connection_refused {
+	unsigned int count;
+};
+
+static void
+test_client_connection_refused_response(
+	const struct http_response *resp,
+	struct _connection_refused *ctx)
+{
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED);
+	test_assert(resp->reason != NULL && *resp->reason != '\0');
+
+	if (--ctx->count == 0) {
+		i_free(ctx);
+		io_loop_stop(ioloop);
+	}
+}
+
+static void
+test_client_connection_refused(const struct http_client_settings *client_set)
+{
+	struct http_client_request *hreq;
+	struct _connection_refused *ctx;
+
+	ctx = i_new(struct _connection_refused, 1);
+	ctx->count = 2;
+
+	http_client = http_client_init(client_set);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/connection-refused.txt",
+		test_client_connection_refused_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_submit(hreq);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/connection-refused2.txt",
+		test_client_connection_refused_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_submit(hreq);
+}
+
+/* test */
+
+static void test_connection_refused(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+
+	test_begin("connection refused");
+	test_run_client_server(&http_client_set,
+		test_client_connection_refused,
+		test_server_connection_refused, 1);
+	test_end();
+}
+
+/*
+ * Connection timed out
+ */
+
+/* client */
+
+struct _connection_timed_out {
+	unsigned int count;
+};
+
+static void
+test_client_connection_timed_out_response(
+	const struct http_response *resp,
+	struct _connection_timed_out *ctx)
+{
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECT_FAILED);
+	test_assert(resp->reason != NULL && *resp->reason != '\0');
+
+	if (--ctx->count == 0) {
+		i_free(ctx);
+		io_loop_stop(ioloop);
+	}
+}
+
+static void
+test_client_connection_timed_out(
+	const struct http_client_settings *client_set)
+{
+	struct http_client_request *hreq;
+	struct _connection_timed_out *ctx;
+
+	ctx = i_new(struct _connection_timed_out, 1);
+	ctx->count = 2;
+
+	http_client = http_client_init(client_set);
+
+	hreq = http_client_request(http_client,
+		"GET", "192.168.0.0", "/connection-timed-out.txt",
+		test_client_connection_timed_out_response, ctx);
+	http_client_request_submit(hreq);
+
+	hreq = http_client_request(http_client,
+		"GET", "192.168.0.0", "/connection-timed-out2.txt",
+		test_client_connection_timed_out_response, ctx);
+	http_client_request_submit(hreq);
+}
+
+/* test */
+
+static void test_connection_timed_out(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+	http_client_set.connect_timeout_msecs = 1000;
+	http_client_set.max_attempts = 1;
+
+	test_begin("connection timed out");
+	test_run_client_server(&http_client_set,
+		test_client_connection_timed_out,
+		NULL, 0);
+	test_end();
+}
+
+/*
+ * Invalid redirect
+ */
+
+/* server */
+
+/* -> not accepted */
+
+static void
+test_invalid_redirect_input1(struct server_connection *conn)
+{
+	static const char *resp =
+		"HTTP/1.1 302 Redirect\r\n"
+		"Location: http://localhost:4444\r\n"
+		"\r\n";
+	o_stream_nsend_str(conn->conn.output, resp);
+	server_connection_deinit(&conn);
+}
+
+static void test_server_invalid_redirect1(unsigned int index)
+{
+	test_server_input = test_invalid_redirect_input1;
+	test_server_run(index);
+}
+
+/* -> bad location */
+
+static void
+test_invalid_redirect_input2(struct server_connection *conn)
+{
+	static const char *resp =
+		"HTTP/1.1 302 Redirect\r\n"
+		"Location: unix:/var/run/dovecot/auth-master\r\n"
+		"\r\n";
+	o_stream_nsend_str(conn->conn.output, resp);
+	server_connection_deinit(&conn);
+}
+
+static void test_server_invalid_redirect2(unsigned int index)
+{
+	test_server_input = test_invalid_redirect_input2;
+	test_server_run(index);
+}
+
+/* -> too many */
+
+static void
+test_invalid_redirect_input3(struct server_connection *conn)
+{
+	string_t *resp;
+
+	resp = t_str_new(512);
+	str_printfa(resp, 
+		"HTTP/1.1 302 Redirect\r\n"
+		"Location: http://%s:%u/friep.txt\r\n"
+		"\r\n",
+		net_ip2addr(&bind_ip), bind_ports[server_index+1]);
+	o_stream_nsend(conn->conn.output,
+		str_data(resp), str_len(resp));
+	server_connection_deinit(&conn);
+}
+
+static void test_server_invalid_redirect3(unsigned int index)
+{
+	test_server_input = test_invalid_redirect_input3;
+	test_server_run(index);
+}
+
+/* client */
+
+static void
+test_client_invalid_redirect_response(
+	const struct http_response *resp,
+	void *context ATTR_UNUSED)
+{
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_INVALID_REDIRECT);
+	test_assert(resp->reason != NULL && *resp->reason != '\0');
+	io_loop_stop(ioloop);
+}
+
+static void
+test_client_invalid_redirect(const struct http_client_settings *client_set)
+{
+	struct http_client_request *hreq;
+
+	http_client = http_client_init(client_set);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/invalid-redirect.txt",
+		test_client_invalid_redirect_response, NULL);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_submit(hreq);
+}
+
+/* test */
+
+static void test_invalid_redirect(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+
+	test_begin("invalid redirect: not accepted");
+	http_client_set.max_redirects = 0;
+	test_run_client_server(&http_client_set,
+		test_client_invalid_redirect,
+		test_server_invalid_redirect1, 1);
+	test_end();
+
+	test_begin("invalid redirect: bad location");
+	http_client_set.max_redirects = 1;
+	test_run_client_server(&http_client_set,
+		test_client_invalid_redirect,
+		test_server_invalid_redirect2, 1);
+	test_end();
+
+	test_begin("invalid redirect: too many");
+	http_client_set.max_redirects = 1;
+	test_run_client_server(&http_client_set,
+		test_client_invalid_redirect,
+		test_server_invalid_redirect3, 3);
+	test_end();
+}
+
+/* 
+ * Unseekable redirect
+ */
+
+/* server */
+
+static void
+test_unseekable_redirect_input(struct server_connection *conn)
+{
+	string_t *resp;
+
+	resp = t_str_new(512);
+	str_printfa(resp, 
+		"HTTP/1.1 302 Redirect\r\n"
+		"Location: http://%s:%u/frml.txt\r\n"
+		"\r\n",
+		net_ip2addr(&bind_ip), bind_ports[server_index+1]);
+	o_stream_nsend(conn->conn.output,
+		str_data(resp), str_len(resp));
+	server_connection_deinit(&conn);
+}
+
+static void test_server_unseekable_redirect(unsigned int index)
+{
+	test_server_input = test_unseekable_redirect_input;
+	test_server_run(index);
+}
+
+/* client */
+
+static void
+test_client_unseekable_redirect_response(
+	const struct http_response *resp,
+	void *context ATTR_UNUSED)
+{
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_ABORTED);
+	test_assert(resp->reason != NULL && *resp->reason != '\0');
+	io_loop_stop(ioloop);
+}
+
+static void
+test_client_unseekable_redirect(const struct http_client_settings *client_set)
+{
+	struct http_client_request *hreq;
+	struct istream *input;
+
+	http_client = http_client_init(client_set);
+
+	input = i_stream_create_from_data("FROP", 4);
+	input->seekable = FALSE;
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/unseekable-redirect.txt",
+		test_client_unseekable_redirect_response, NULL);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_set_payload(hreq, input, FALSE);
+	http_client_request_submit(hreq);
+
+	i_stream_unref(&input);	
+}
+
+/* test */
+
+static void test_unseekable_redirect(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+	http_client_set.max_redirects = 1;
+
+	test_begin("unseekable redirect");
+	test_run_client_server(&http_client_set,
+		test_client_unseekable_redirect,
+		test_server_unseekable_redirect, 2);
+	test_end();
+}
+
+/*
+ * Unseekable retry
+ */
+
+/* server */
+
+static void
+test_unseekable_retry_input(struct server_connection *conn)
+{
+	server_connection_deinit(&conn);
+}
+
+static void test_server_unseekable_retry(unsigned int index)
+{
+	test_server_input = test_unseekable_retry_input;
+	test_server_run(index);
+}
+
+/* client */
+
+static void
+test_client_unseekable_retry_response(
+	const struct http_response *resp,
+	void *context ATTR_UNUSED)
+{
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_ABORTED);
+	test_assert(resp->reason != NULL && *resp->reason != '\0');
+	io_loop_stop(ioloop);
+}
+
+static void
+test_client_unseekable_retry(const struct http_client_settings *client_set)
+{
+	struct http_client_request *hreq;
+	struct istream *input;
+
+	http_client = http_client_init(client_set);
+
+	input = i_stream_create_from_data("FROP", 4);
+	input->seekable = FALSE;
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/unseekable-retry.txt",
+		test_client_unseekable_retry_response, NULL);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_set_payload(hreq, input, FALSE);
+	http_client_request_submit(hreq);
+
+	i_stream_unref(&input);	
+}
+
+/* test */
+
+static void test_unseekable_retry(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+	http_client_set.max_attempts = 3;
+
+	test_begin("unseekable retry");
+	test_run_client_server(&http_client_set,
+		test_client_unseekable_retry,
+		test_server_unseekable_retry, 2);
+	test_end();
+}
+
+/*
+ * Broken payload
+ */
+
+/* server */
+
+static void
+test_broken_payload_input(struct server_connection *conn)
+{
+	static const char *resp =
+		"HTTP/1.1 200 OK\r\n"
+		"Content-Length: 18\r\n"
+		"\r\n"
+		"Everything is OK\r\n";
+	o_stream_nsend_str(conn->conn.output, resp);
+	server_connection_deinit(&conn);
+}
+
+static void test_server_broken_payload(unsigned int index)
+{
+	test_server_input = test_broken_payload_input;
+	test_server_run(index);
+}
+
+/* client */
+
+static void
+test_client_broken_payload_response(
+	const struct http_response *resp,
+	void *context ATTR_UNUSED)
+{
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_BROKEN_PAYLOAD);
+	test_assert(resp->reason != NULL && *resp->reason != '\0');
+	io_loop_stop(ioloop);
+}
+
+static void
+test_client_broken_payload(const struct http_client_settings *client_set)
+{
+	struct http_client_request *hreq;
+	struct istream *input;
+
+	test_expect_errors(1);
+
+	http_client = http_client_init(client_set);
+
+	input = i_stream_create_error_str(EIO, "Moehahahaha!!");
+	i_stream_set_name(input, "PURE EVIL");
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/broken-payload.txt",
+		test_client_broken_payload_response, NULL);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_set_payload(hreq, input, FALSE);
+	http_client_request_submit(hreq);
+
+	i_stream_unref(&input);	
+}
+
+/* test */
+
+static void test_broken_payload(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+
+	test_begin("broken payload");
+	test_run_client_server(&http_client_set,
+		test_client_broken_payload,
+		test_server_broken_payload, 1);
+	test_end();
+}
+
+/*
+ * Connection lost
+ */
+
+/* server */
+
+static void
+test_connection_lost_input(struct server_connection *conn)
+{
+	ssize_t ret;
+
+	if (server_read_max == 0) {
+		server_connection_deinit(&conn);
+		return;
+	}
+
+	i_stream_set_max_buffer_size(conn->conn.input, server_read_max);
+	ret = i_stream_read(conn->conn.input);
+	if (ret == -2) {
+		server_connection_deinit(&conn);
+		return;
+	}
+	if (ret < 0) {
+		if (i_stream_is_eof(conn->conn.input))
+			i_fatal("server: Client stream ended prematurely");
+		else
+			i_fatal("server: Streem error: %s",
+				i_stream_get_error(conn->conn.input));
+	}
+}
+
+static void test_server_connection_lost(unsigned int index)
+{
+	test_server_input = test_connection_lost_input;
+	test_server_run(index);
+}
+
+/* client */
+
+struct _connection_lost_ctx {
+	unsigned int count;
+};
+
+static void
+test_client_connection_lost_response(
+	const struct http_response *resp,
+	struct _connection_lost_ctx *ctx)
+{
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST);
+	test_assert(resp->reason != NULL && *resp->reason != '\0');
+
+	if (--ctx->count == 0) {
+		i_free(ctx);
+		io_loop_stop(ioloop);
+	}
+}
+
+static void
+test_client_connection_lost(const struct http_client_settings *client_set)
+{
+	static const char payload[] =
+		"This is a useless payload that only serves as a means to give the "
+		"server the opportunity to close the connection before the payload "
+		"is finished.";
+	struct _connection_lost_ctx *ctx;
+	struct http_client_request *hreq;
+	struct istream *input;
+
+	ctx = i_new(struct _connection_lost_ctx, 1);
+	ctx->count = 2;
+
+	http_client = http_client_init(client_set);
+
+	input = i_stream_create_from_data(payload, sizeof(payload)-1);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/connection-lost.txt",
+		test_client_connection_lost_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_set_payload(hreq, input, FALSE);
+	http_client_request_submit(hreq);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/connection-lost2.txt",
+		test_client_connection_lost_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_submit(hreq);
+
+	i_stream_unref(&input);	
+}
+
+/* test */
+
+static void test_connection_lost(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+
+	server_read_max = 0;
+
+	test_begin("connection lost: one attempt");
+	http_client_set.max_attempts = 1;
+	test_run_client_server(&http_client_set,
+		test_client_connection_lost,
+		test_server_connection_lost, 1);
+	test_end();
+
+	test_begin("connection lost: two attempts");
+	http_client_set.max_attempts = 2;
+	test_run_client_server(&http_client_set,
+		test_client_connection_lost,
+		test_server_connection_lost, 1);
+	test_end();
+
+	test_begin("connection lost: three attempts");
+	http_client_set.max_attempts = 3;
+	test_run_client_server(&http_client_set,
+		test_client_connection_lost,
+		test_server_connection_lost, 1);
+	test_end();
+}
+
+/*
+ * Connection lost after 100-continue
+ */
+
+/* server */
+
+static void
+test_connection_lost_100_input(struct server_connection *conn)
+{
+	static const char *resp =
+		"HTTP/1.1 100 Continue\r\n"
+		"\r\n";
+	o_stream_nsend_str(conn->conn.output, resp);
+	server_connection_deinit(&conn);
+}
+
+static void test_server_connection_lost_100(unsigned int index)
+{
+	test_server_input = test_connection_lost_100_input;
+	test_server_run(index);
+}
+
+/* client */
+
+struct _connection_lost_100_ctx {
+	unsigned int count;
+};
+
+static void
+test_client_connection_lost_100_response(
+	const struct http_response *resp,
+	struct _connection_lost_100_ctx *ctx)
+{
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST);
+	test_assert(resp->reason != NULL && *resp->reason != '\0');
+
+	if (--ctx->count == 0) {
+		i_free(ctx);
+		io_loop_stop(ioloop);
+	}
+}
+
+static void
+test_client_connection_lost_100(
+	const struct http_client_settings *client_set)
+{
+	static const char payload[] =
+		"This is a useless payload that only serves as a means to give the "
+		"server the opportunity to close the connection before the payload "
+		"is finished.";
+	struct _connection_lost_100_ctx *ctx;
+	struct http_client_request *hreq;
+	struct istream *input;
+
+	ctx = i_new(struct _connection_lost_100_ctx, 1);
+	ctx->count = 2;
+
+	http_client = http_client_init(client_set);
+
+	input = i_stream_create_from_data(payload, sizeof(payload)-1);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/connection-lost.txt",
+		test_client_connection_lost_100_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_set_payload(hreq, input, TRUE);
+	http_client_request_submit(hreq);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/connection-lost2.txt",
+		test_client_connection_lost_100_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_set_payload(hreq, input, TRUE);
+	http_client_request_submit(hreq);
+
+	i_stream_unref(&input);	
+}
+
+/* test */
+
+static void test_connection_lost_100(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+
+	server_read_max = 0;
+
+	test_begin("connection lost after 100-continue");
+	http_client_set.max_attempts = 1;
+	test_run_client_server(&http_client_set,
+		test_client_connection_lost_100,
+		test_server_connection_lost_100, 1);
+	test_end();
+}
+
+/*
+ * Connection lost in sub-ioloop
+ */
+
+/* server */
+
+static void
+test_connection_lost_sub_ioloop_input(struct server_connection *conn)
+{
+	static const char *resp =
+		"HTTP/1.1 200 OK\r\n"
+		"Content-Length: 0\r\n"
+		"\r\n";
+	o_stream_nsend_str(conn->conn.output, resp);
+	server_connection_deinit(&conn);
+}
+
+static void test_server_connection_lost_sub_ioloop(unsigned int index)
+{
+	test_server_input = test_connection_lost_sub_ioloop_input;
+	test_server_run(index);
+}
+
+/* client */
+
+struct _connection_lost_sub_ioloop_ctx {
+	unsigned int count;
+};
+
+static void
+test_client_connection_lost_sub_ioloop_response2(
+	const struct http_response *resp,
+	struct ioloop *sub_ioloop)
+{
+	if (debug)
+		i_debug("SUB-RESPONSE: %u %s", resp->status, resp->reason);
+	io_loop_stop(sub_ioloop);
+}
+
+static void
+test_client_connection_lost_sub_ioloop_response(
+	const struct http_response *resp,
+	struct _connection_lost_sub_ioloop_ctx *ctx)
+{
+	struct http_client_request *hreq;
+	struct ioloop *sub_ioloop;
+
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	test_assert(resp->status == 200);
+	test_assert(resp->reason != NULL && *resp->reason != '\0');
+
+	sub_ioloop = io_loop_create();
+	http_client_switch_ioloop(http_client);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/connection-lost-sub-ioloop3.txt",
+		test_client_connection_lost_sub_ioloop_response2, sub_ioloop);
+	http_client_request_set_port(hreq, bind_ports[1]);
+	http_client_request_submit(hreq);
+
+	io_loop_run(sub_ioloop);
+	io_loop_set_current(ioloop);
+	http_client_switch_ioloop(http_client);
+	io_loop_set_current(sub_ioloop);
+	io_loop_destroy(&sub_ioloop);
+
+	if (--ctx->count == 0) {
+		i_free(ctx);
+		io_loop_stop(ioloop);
+	}
+}
+
+static void
+test_client_connection_lost_sub_ioloop(
+	const struct http_client_settings *client_set)
+{
+	static const char payload[] =
+		"This is a useless payload that only serves as a means to give the "
+		"server the opportunity to close the connection before the payload "
+		"is finished.";
+	struct _connection_lost_sub_ioloop_ctx *ctx;
+	struct http_client_request *hreq;
+	struct istream *input;
+
+	ctx = i_new(struct _connection_lost_sub_ioloop_ctx, 1);
+	ctx->count = 2;
+
+	http_client = http_client_init(client_set);
+
+	input = i_stream_create_from_data(payload, sizeof(payload)-1);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/connection-lost-sub-ioloop.txt",
+		test_client_connection_lost_sub_ioloop_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_set_payload(hreq, input, TRUE);
+	http_client_request_submit(hreq);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/connection-lost-sub-ioloop2.txt",
+		test_client_connection_lost_sub_ioloop_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_set_payload(hreq, input, TRUE);
+	http_client_request_submit(hreq);
+
+	i_stream_unref(&input);	
+}
+
+/* test */
+
+static void test_connection_lost_sub_ioloop(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+
+	server_read_max = 0;
+
+	test_begin("connection lost while running sub-ioloop");
+	http_client_set.max_attempts = 1;
+	test_run_client_server(&http_client_set,
+		test_client_connection_lost_sub_ioloop,
+		test_server_connection_lost_sub_ioloop, 2);
+	test_end();
+}
+
+/*
+ * Early success
+ */
+
+/* server */
+
+static void
+test_early_success_input(struct server_connection *conn)
+{
+	static const char *resp =
+		"HTTP/1.1 200 OK\r\n"
+		"Content-Length: 18\r\n"
+		"\r\n"
+		"Everything is OK\r\n";
+	
+	usleep(200000);
+	o_stream_nsend_str(conn->conn.output, resp);
+	server_connection_deinit(&conn);
+}
+
+static void test_server_early_success(unsigned int index)
+{
+	test_server_input = test_early_success_input;
+	test_server_run(index);
+}
+
+/* client */
+
+struct _early_success_ctx {
+	unsigned int count;
+};
+
+static void
+test_client_early_success_response(
+	const struct http_response *resp,
+	struct _early_success_ctx *ctx)
+{
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	if (ctx->count == 2)
+		test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE);
+	else
+		test_assert(resp->status == 200);
+	test_assert(resp->reason != NULL && *resp->reason != '\0');
+	if (--ctx->count == 0) {
+		io_loop_stop(ioloop);
+		i_free(ctx);
+	}
+}
+
+static void
+test_client_early_success(const struct http_client_settings *client_set)
+{
+	struct http_client_request *hreq;
+	struct _early_success_ctx *ctx;
+	string_t *payload;
+	unsigned int i;
+
+	ctx = i_new(struct _early_success_ctx, 1);
+	ctx->count = 2;
+	
+	http_client = http_client_init(client_set);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/early-success.txt",
+		test_client_early_success_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+
+	T_BEGIN {
+		payload = t_str_new(204800);
+		for (i = 0; i < 3200; i++) {
+			str_append(payload,
+				"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n"
+				"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n");
+		}
+
+		http_client_request_set_payload_data
+			(hreq, str_data(payload), str_len(payload));
+	} T_END;
+	http_client_request_submit(hreq);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/early-success2.txt",
+		test_client_early_success_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_submit(hreq);
+}
+
+/* test */
+
+static void test_early_success(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+	http_client_set.socket_send_buffer_size = 4096;
+
+	test_begin("early succes");
+	test_run_client_server(&http_client_set,
+		test_client_early_success,
+		test_server_early_success, 1);
+	test_end();
+}
+
+/*
+ * Bad response
+ */
+
+/* server */
+
+static void
+test_bad_response_input(struct server_connection *conn)
+{
+	static const char *resp =
+		"HTTP/1.1 666 Really bad response\r\n"
+		"\r\n";
+	o_stream_nsend_str(conn->conn.output, resp);
+	server_connection_deinit(&conn);
+}
+
+static void test_server_bad_response(unsigned int index)
+{
+	test_server_input = test_bad_response_input;
+	test_server_run(index);
+}
+
+/* client */
+
+struct _bad_response_ctx {
+	unsigned int count;
+};
+
+static void
+test_client_bad_response_response(
+	const struct http_response *resp,
+	struct _bad_response_ctx *ctx)
+{
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_BAD_RESPONSE);
+	test_assert(resp->reason != NULL && *resp->reason != '\0');
+
+	if (--ctx->count == 0) {
+		i_free(ctx);
+		io_loop_stop(ioloop);
+	}
+}
+
+static void
+test_client_bad_response(const struct http_client_settings *client_set)
+{
+	struct http_client_request *hreq;
+	struct _bad_response_ctx *ctx;
+
+	ctx = i_new(struct _bad_response_ctx, 1);
+	ctx->count = 2;
+
+	http_client = http_client_init(client_set);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/bad-response.txt",
+		test_client_bad_response_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_submit(hreq);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/bad-response2.txt",
+		test_client_bad_response_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_submit(hreq);
+}
+
+/* test */
+
+static void test_bad_response(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+
+	test_begin("bad response");
+	test_run_client_server(&http_client_set,
+		test_client_bad_response,
+		test_server_bad_response, 1);
+	test_end();
+}
+
+/*
+ * Request timed out
+ */
+
+/* server */
+
+static void
+test_request_timed_out_input(struct server_connection *conn ATTR_UNUSED)
+{
+	/* do nothing */
+}
+
+static void test_server_request_timed_out(unsigned int index)
+{
+	test_server_input = test_request_timed_out_input;
+	test_server_run(index);
+}
+
+/* client */
+
+struct _request_timed_out_ctx {
+	unsigned int count;
+};
+
+static void
+test_client_request_timed_out_response(
+	const struct http_response *resp,
+	struct _request_timed_out_ctx *ctx)
+{
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_TIMED_OUT);
+	test_assert(resp->reason != NULL && *resp->reason != '\0');
+
+	if (--ctx->count == 0) {
+		i_free(ctx);
+		io_loop_stop(ioloop);
+	}
+}
+
+static void
+test_client_request_timed_out(const struct http_client_settings *client_set)
+{
+	struct http_client_request *hreq;
+	struct _request_timed_out_ctx *ctx;
+
+	ctx = i_new(struct _request_timed_out_ctx, 1);
+	ctx->count = 2;
+
+	http_client = http_client_init(client_set);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/request-timed-out.txt",
+		test_client_request_timed_out_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_submit(hreq);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/request-timed-out2.txt",
+		test_client_request_timed_out_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_submit(hreq);
+}
+
+/* test */
+
+static void test_request_timed_out(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+
+	test_begin("request timed out: one attempt");
+	http_client_set.request_timeout_msecs = 1000;
+	http_client_set.max_attempts = 1;
+	test_run_client_server(&http_client_set,
+		test_client_request_timed_out,
+		test_server_request_timed_out, 1);
+	test_end();
+
+	test_begin("request timed out: two attempts");
+	http_client_set.request_timeout_msecs = 1000;
+	http_client_set.max_attempts = 1;
+	test_run_client_server(&http_client_set,
+		test_client_request_timed_out,
+		test_server_request_timed_out, 1);
+	test_end();
+
+	test_begin("request absolutely timed out");
+	http_client_set.request_timeout_msecs = 0;
+	http_client_set.request_absolute_timeout_msecs = 2000;
+	http_client_set.max_attempts = 3;
+	test_run_client_server(&http_client_set,
+		test_client_request_timed_out,
+		test_server_request_timed_out, 1);
+	test_end();
+
+	test_begin("request double timed out");
+	http_client_set.request_timeout_msecs = 500;
+	http_client_set.request_absolute_timeout_msecs = 2000;
+	http_client_set.max_attempts = 3;
+	test_run_client_server(&http_client_set,
+		test_client_request_timed_out,
+		test_server_request_timed_out, 1);
+	test_end();
+}
+
+/*
+ * Request aborted early
+ */
+
+/* server */
+
+static void
+test_request_aborted_early_input(struct server_connection *conn ATTR_UNUSED)
+{
+	static const char *resp =
+		"HTTP/1.1 404 Not Found\r\n"
+		"\r\n";
+
+	/* wait one second to respon */
+	sleep(1);
+
+	/* respond */
+	o_stream_nsend_str(conn->conn.output, resp);
+	server_connection_deinit(&conn);
+}
+
+static void test_server_request_aborted_early(unsigned int index)
+{
+	test_server_input = test_request_aborted_early_input;
+	test_server_run(index);
+}
+
+/* client */
+
+struct _request_aborted_early_ctx {
+	struct http_client_request *req1, *req2;
+	struct timeout *to;
+};
+
+static void
+test_client_request_aborted_early_response(
+	const struct http_response *resp,
+	struct _request_aborted_early_ctx *ctx ATTR_UNUSED)
+{
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	/* abort does not trigger callback */
+	test_assert(FALSE); 
+}
+
+static void
+test_client_request_aborted_early_timeout(
+	struct _request_aborted_early_ctx *ctx)
+{
+	timeout_remove(&ctx->to);
+
+	if (ctx->req1 != NULL) {
+		/* abort early */
+		http_client_request_abort(&ctx->req1); /* sent */
+		http_client_request_abort(&ctx->req2); /* only queued */
+	
+		/* wait a little for server to actually respond to an
+		   already aborted request */
+		ctx->to = timeout_add_short(1000,
+			test_client_request_aborted_early_timeout, ctx);
+	} else {
+		/* all done */
+		i_free(ctx);
+		io_loop_stop(ioloop);
+	}
+}
+
+static void
+test_client_request_aborted_early(
+	const struct http_client_settings *client_set)
+{
+	struct http_client_request *hreq;
+	struct _request_aborted_early_ctx *ctx;
+
+	ctx = i_new(struct _request_aborted_early_ctx, 1);
+
+	http_client = http_client_init(client_set);
+
+	hreq = ctx->req1 = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/request-aborted-early.txt",
+		test_client_request_aborted_early_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_submit(hreq);
+
+	hreq = ctx->req2 = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/request-aborted-early2.txt",
+		test_client_request_aborted_early_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_submit(hreq);
+
+	ctx->to = timeout_add_short(500,
+		test_client_request_aborted_early_timeout, ctx);
+}
+
+/* test */
+
+static void test_request_aborted_early(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+
+	test_begin("request aborted early");
+	test_run_client_server(&http_client_set,
+		test_client_request_aborted_early,
+		test_server_request_aborted_early, 1);
+	test_end();
+}
+
+/*
+ * Client deinit early
+ */
+
+/* server */
+
+static void
+test_client_deinit_early_input(struct server_connection *conn ATTR_UNUSED)
+{
+	static const char *resp =
+		"HTTP/1.1 404 Not Found\r\n"
+		"\r\n";
+
+	/* wait one second to respon */
+	sleep(1);
+
+	/* respond */
+	o_stream_nsend_str(conn->conn.output, resp);
+	server_connection_deinit(&conn);
+}
+
+static void test_server_client_deinit_early(unsigned int index)
+{
+	test_server_input = test_client_deinit_early_input;
+	test_server_run(index);
+}
+
+/* client */
+
+struct _client_deinit_early_ctx {
+	struct timeout *to;
+};
+
+static void
+test_client_client_deinit_early_response(
+	const struct http_response *resp,
+	struct _client_deinit_early_ctx *ctx ATTR_UNUSED)
+{
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	/* abort does not trigger callback */
+	test_assert(FALSE); 
+}
+
+static void
+test_client_client_deinit_early_timeout(
+	struct _client_deinit_early_ctx *ctx)
+{
+	timeout_remove(&ctx->to);
+
+	/* deinit early */
+	http_client_deinit(&http_client);
+	
+	/* all done */
+	i_free(ctx);
+	io_loop_stop(ioloop);
+}
+
+static void
+test_client_client_deinit_early(const struct http_client_settings *client_set)
+{
+	struct http_client_request *hreq;
+	struct _client_deinit_early_ctx *ctx;
+
+	ctx = i_new(struct _client_deinit_early_ctx, 1);
+
+	http_client = http_client_init(client_set);
+
+	hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/client-deinit-early.txt",
+		test_client_client_deinit_early_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_submit(hreq);
+
+	hreq =  http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/client-deinit-early2.txt",
+		test_client_client_deinit_early_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_submit(hreq);
+
+	ctx->to = timeout_add_short(500,
+		test_client_client_deinit_early_timeout, ctx);
+}
+
+/* test */
+
+static void test_client_deinit_early(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+
+	test_begin("client deinit early");
+	test_run_client_server(&http_client_set,
+		test_client_client_deinit_early,
+		test_server_client_deinit_early, 1);
+	test_end();
+}
+
+/*
+ * All tests
+ */
+
+static void (*test_functions[])(void) = {
+	test_host_lookup_failed,
+	test_connection_refused,
+	test_connection_timed_out,
+	test_invalid_redirect,
+	test_unseekable_redirect,
+	test_unseekable_retry,
+	test_broken_payload,
+	test_connection_lost,
+	test_connection_lost_100,
+	test_connection_lost_sub_ioloop,
+	test_early_success,
+	test_bad_response,
+	test_request_timed_out,
+	test_request_aborted_early,
+	test_client_deinit_early,
+	NULL
+};
+
+/*
+ * Test client
+ */
+
+static void
+test_client_defaults(struct http_client_settings *http_set)
+{
+	/* client settings */
+	memset(http_set, 0, sizeof(*http_set));
+	http_set->max_idle_time_msecs = 5*1000;
+	http_set->max_parallel_connections = 1;
+	http_set->max_pipelined_requests = 1;
+	http_set->max_redirects = 0;
+	http_set->max_attempts = 1;
+	http_set->debug = debug;
+}
+
+static void test_client_deinit(void)
+{
+	if (http_client != NULL)
+		http_client_deinit(&http_client);
+	http_client = NULL;
+}
+
+/*
+ * Test server
+ */
+
+/* client connection */
+
+static void
+server_connection_input(struct connection *_conn)
+{
+	struct server_connection *conn = (struct server_connection *)_conn;
+	
+	test_server_input(conn);
+}
+
+static void
+server_connection_init(int fd)
+{
+	struct server_connection *conn;
+	pool_t pool;
+
+	net_set_nonblock(fd, TRUE);
+
+	pool = pool_alloconly_create("server connection", 256);
+	conn = p_new(pool, struct server_connection, 1);
+	conn->pool = pool;
+
+	connection_init_server
+		(server_conn_list, &conn->conn, "server connection", fd, fd);
+}
+
+static void
+server_connection_deinit(struct server_connection **_conn)
+{
+	struct server_connection *conn = *_conn;
+
+	*_conn = NULL;
+
+	connection_deinit(&conn->conn);
+	pool_unref(&conn->pool);
+}
+
+static void
+server_connection_destroy(struct connection *_conn)
+{
+	struct server_connection *conn =
+		(struct server_connection *)_conn;
+
+	server_connection_deinit(&conn);
+}
+
+static void
+server_connection_accept(void *context ATTR_UNUSED)
+{
+	int fd;
+
+	/* accept new client */
+	fd = net_accept(fd_listen, NULL, NULL);
+	if (fd == -1)
+		return;
+	if (fd == -2) {
+		i_fatal("test server: accept() failed: %m");
+	}
+
+	server_connection_init(fd);
+}
+
+/* */
+
+static struct connection_settings server_connection_set = {
+	.input_max_size = (size_t)-1,
+	.output_max_size = (size_t)-1,
+	.client = FALSE
+};
+
+static const struct connection_vfuncs server_connection_vfuncs = {
+	.destroy = server_connection_destroy,
+	.input = server_connection_input
+};
+
+static void test_server_run(unsigned int index)
+{
+	server_index = index;
+
+	/* open server socket */
+	io_listen = io_add(fd_listen,
+		IO_READ, server_connection_accept, (void *)NULL);
+
+	server_conn_list = connection_list_init
+		(&server_connection_set, &server_connection_vfuncs);
+
+	io_loop_run(ioloop);
+
+	/* close server socket */
+	io_remove(&io_listen);
+
+	connection_list_deinit(&server_conn_list);
+}
+
+/*
+ * Tests
+ */
+
+static int test_open_server_fd(in_port_t *bind_port)
+{
+	int fd = net_listen(&bind_ip, bind_port, 128);
+	if (debug)
+		i_debug("server listening on %u", *bind_port);
+	if (fd == -1) {
+		i_fatal("listen(%s:%u) failed: %m",
+			net_ip2addr(&bind_ip), *bind_port);
+	}
+	return fd;
+}
+
+static void test_servers_kill_all(void)
+{
+	unsigned int i;
+
+	if (server_pids_count > 0) {
+		for (i = 0; i < server_pids_count; i++) {
+			if (server_pids[i] != (pid_t)-1) {
+				(void)kill(server_pids[i], SIGKILL);
+				(void)waitpid(server_pids[i], NULL, 0);
+				server_pids[i] = -1;
+			}
+		}
+	}
+	server_pids_count = 0;
+}
+
+static void test_run_client_server(
+	const struct http_client_settings *client_set,
+	test_client_init_t client_test,
+	test_server_init_t server_test,
+	unsigned int server_tests_count)
+{
+	unsigned int i;
+
+	server_pids = NULL;
+	server_pids_count = 0;
+
+	if (server_tests_count > 0) {
+		int fds[server_tests_count];
+
+		bind_ports = i_new(in_port_t, server_tests_count);
+
+		server_pids = i_new(pid_t, server_tests_count);
+		for (i = 0; i < server_tests_count; i++)
+			server_pids[i] = (pid_t)-1;
+		server_pids_count = server_tests_count;
+
+		for (i = 0; i < server_tests_count; i++)
+			fds[i] = test_open_server_fd(&bind_ports[i]);
+
+		for (i = 0; i < server_tests_count; i++) {
+			fd_listen = fds[i];
+			if ((server_pids[i] = fork()) == (pid_t)-1)
+				i_fatal("fork() failed: %m");
+			if (server_pids[i] == 0) {
+				server_pids[i] = (pid_t)-1;
+				server_pids_count = 0;
+				hostpid_init();
+				if (debug)
+					i_debug("server[%d]: PID=%s", i+1, my_pid);
+				/* child: server */
+				ioloop = io_loop_create();
+				server_test(i);
+				io_loop_destroy(&ioloop);
+				if (fd_listen != -1)
+					i_close_fd(&fd_listen);
+				i_free(bind_ports);
+				i_free(server_pids);
+				/* wait for it to be killed; this way, valgrind will not
+				   object to this process going away inelegantly. */
+				sleep(60);
+				exit(1);
+			}
+			if (fd_listen != -1)
+				i_close_fd(&fd_listen);
+		}
+		if (debug)
+			i_debug("client: PID=%s", my_pid);
+	}
+
+	/* parent: client */
+
+	usleep(100000); /* wait a little for server setup */
+
+	ioloop = io_loop_create();
+	client_test(client_set);
+	io_loop_run(ioloop);
+	test_client_deinit();
+	io_loop_destroy(&ioloop);
+
+	test_servers_kill_all();
+	i_free(server_pids);
+	i_free(bind_ports);
+}
+
+/*
+ * Main
+ */
+
+volatile sig_atomic_t terminating = 0;
+
+static void
+test_signal_handler(int signo)
+{
+	if (terminating)
+		raise(signo);
+	terminating = 1;
+
+	/* make sure we don't leave any pesky children alive */
+	test_servers_kill_all();
+
+	(void)signal(signo, SIG_DFL);
+	raise(signo);
+}
+
+static void test_atexit(void)
+{
+	test_servers_kill_all();
+}
+
+int main(int argc, char *argv[])
+{
+	int c;
+
+	atexit(test_atexit);
+	(void)signal(SIGCHLD, SIG_IGN);
+	(void)signal(SIGTERM, test_signal_handler);
+	(void)signal(SIGQUIT, test_signal_handler);
+	(void)signal(SIGINT, test_signal_handler);
+	(void)signal(SIGSEGV, test_signal_handler);
+	(void)signal(SIGABRT, test_signal_handler);
+
+  while ((c = getopt(argc, argv, "D")) > 0) {
+		switch (c) {
+		case 'D':
+			debug = TRUE;
+			break;
+		default:
+			i_fatal("Usage: %s [-D]", argv[0]);
+		}
+  }
+
+	/* listen on localhost */
+	memset(&bind_ip, 0, sizeof(bind_ip));
+	bind_ip.family = AF_INET;
+	bind_ip.u.ip4.s_addr = htonl(INADDR_LOOPBACK);	
+
+	test_run(test_functions);
+}

From 024ade62275e0960cf52f117f20ee7048e21e4c8 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 26 May 2016 18:27:27 +0300
Subject: [PATCH 333/450] lib-http: Improved error logging for
 net_set_*_buffer_size() failures.

---
 src/lib-http/http-client-connection.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/lib-http/http-client-connection.c b/src/lib-http/http-client-connection.c
index 880e89a101e..9b8a6c3e055 100644
--- a/src/lib-http/http-client-connection.c
+++ b/src/lib-http/http-client-connection.c
@@ -1199,12 +1199,14 @@ http_client_connection_connected(struct connection *_conn, bool success)
 		if (set->socket_send_buffer_size > 0) {
 			if (net_set_send_buffer_size(_conn->fd_out,
 				set->socket_send_buffer_size) < 0)
-				i_error("net_set_send_buffer_size() failed: %m");
+				i_error("net_set_send_buffer_size(%"PRIuSIZE_T") failed: %m",
+					set->socket_send_buffer_size);
 		}
 		if (set->socket_recv_buffer_size > 0) {
 			if (net_set_recv_buffer_size(_conn->fd_in,
 				set->socket_recv_buffer_size) < 0)
-				i_error("net_set_recv_buffer_size() failed: %m");
+				i_error("net_set_recv_buffer_size(%"PRIuSIZE_T") failed: %m",
+					set->socket_recv_buffer_size);
 		}
 
 		if (http_client_peer_addr_is_https(&conn->peer->addr)) {

From 91ebc77e7efa3b26ec68214e1b0884ffe37849af Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Tue, 14 Jun 2016 22:47:31 +0200
Subject: [PATCH 334/450] lib-http: client: Added test for
 http_client_request_delay_msecs() in test_http_client_errors.

---
 src/lib-http/test-http-client-errors.c | 106 +++++++++++++++++++++++++
 1 file changed, 106 insertions(+)

diff --git a/src/lib-http/test-http-client-errors.c b/src/lib-http/test-http-client-errors.c
index 36f88891c42..4af94e94f9c 100644
--- a/src/lib-http/test-http-client-errors.c
+++ b/src/lib-http/test-http-client-errors.c
@@ -6,6 +6,7 @@
 #include "ioloop.h"
 #include "istream.h"
 #include "ostream.h"
+#include "time-util.h"
 #include "connection.h"
 #include "test-common.h"
 #include "http-url.h"
@@ -1495,6 +1496,110 @@ static void test_client_deinit_early(void)
 	test_end();
 }
 
+/*
+ * Retry with delay
+ */
+
+/* server */
+
+static void
+test_retry_with_delay_input(struct server_connection *conn)
+{
+	string_t *resp;
+
+	resp = t_str_new(512);
+	str_printfa(resp, 
+		"HTTP/1.1 500 Internal Server Error\r\n"
+		"\r\n");
+	o_stream_nsend(conn->conn.output,
+		str_data(resp), str_len(resp));
+	server_connection_deinit(&conn);
+}
+
+static void test_server_retry_with_delay(unsigned int index)
+{
+	test_server_input = test_retry_with_delay_input;
+	test_server_run(index);
+}
+
+/* client */
+
+struct _client_retry_with_delay_ctx {
+	struct http_client_request *req;
+	unsigned int retries;
+	struct timeval time;
+};
+
+static void
+test_client_retry_with_delay_response(
+	const struct http_response *resp,
+	struct _client_retry_with_delay_ctx *ctx)
+{
+	int real_delay, exp_delay;
+
+	if (debug)
+		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
+
+	test_assert(resp->status == 500);
+	test_assert(resp->reason != NULL && *resp->reason != '\0');
+
+	if (ctx->retries > 0) {
+		/* check delay */
+		real_delay = timeval_diff_msecs(&ioloop_timeval, &ctx->time);
+		exp_delay = (1 << (ctx->retries-1)) * 50;
+		if (real_delay < exp_delay) {
+			i_fatal("Retry delay is too short %d < %d",
+				real_delay, exp_delay);
+		}
+	}
+
+	http_client_request_delay_msecs(ctx->req, (1 << ctx->retries) * 50);
+	ctx->time = ioloop_timeval;
+	if (http_client_request_try_retry(ctx->req)) {
+		ctx->retries++;
+		if (debug)
+			i_debug("retrying");
+		return;
+	}
+
+	i_free(ctx);
+	io_loop_stop(ioloop);
+}
+
+static void
+test_client_retry_with_delay(const struct http_client_settings *client_set)
+{
+	struct http_client_request *hreq;
+	struct _client_retry_with_delay_ctx *ctx;
+
+	ctx = i_new(struct _client_retry_with_delay_ctx, 1);
+	ctx->time = ioloop_timeval;
+
+	http_client = http_client_init(client_set);
+
+	ctx->req = hreq = http_client_request(http_client,
+		"GET", net_ip2addr(&bind_ip), "/retry-with-delay.txt",
+		test_client_retry_with_delay_response, ctx);
+	http_client_request_set_port(hreq, bind_ports[0]);
+	http_client_request_submit(hreq);
+}
+
+/* test */
+
+static void test_retry_with_delay(void)
+{
+	struct http_client_settings http_client_set;
+
+	test_client_defaults(&http_client_set);
+	http_client_set.max_attempts = 3;
+
+	test_begin("retry with delay");
+	test_run_client_server(&http_client_set,
+		test_client_retry_with_delay,
+		test_server_retry_with_delay, 1);
+	test_end();
+}
+
 /*
  * All tests
  */
@@ -1515,6 +1620,7 @@ static void (*test_functions[])(void) = {
 	test_request_timed_out,
 	test_request_aborted_early,
 	test_client_deinit_early,
+	test_retry_with_delay,
 	NULL
 };
 

From ce91fd50870c729c0571c089c7a980fe199e7c25 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Fri, 17 Jun 2016 15:39:36 +0200
Subject: [PATCH 335/450] lib-http: client: Allow retrying requests that failed
 internally.

---
 src/lib-http/http-client-request.c | 29 ++++++++++++++++++++---------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/src/lib-http/http-client-request.c b/src/lib-http/http-client-request.c
index 18222d89666..f1041d7b1a0 100644
--- a/src/lib-http/http-client-request.c
+++ b/src/lib-http/http-client-request.c
@@ -55,7 +55,7 @@ http_client_request_debug(struct http_client_request *req,
  * Request
  */
 
-static void
+static bool
 http_client_request_send_error(struct http_client_request *req,
 			       unsigned int status, const char *error);
 
@@ -1071,12 +1071,13 @@ bool http_client_request_callback(struct http_client_request *req,
 	return TRUE;
 }
 
-static void
+static bool
 http_client_request_send_error(struct http_client_request *req,
 			       unsigned int status, const char *error)
 {
 	http_client_request_callback_t *callback;
 	bool sending = (req->state == HTTP_REQUEST_STATE_PAYLOAD_OUT);
+	unsigned int orig_attempts = req->attempts;
 
 	req->state = HTTP_REQUEST_STATE_ABORTED;
 
@@ -1088,28 +1089,38 @@ http_client_request_send_error(struct http_client_request *req,
 		http_response_init(&response, status, error);
 		(void)callback(&response, req->context);
 
-		/* release payload early (prevents server/client deadlock in proxy) */
-		if (!sending && req->payload_input != NULL)
-			i_stream_unref(&req->payload_input);
+		if (req->attempts != orig_attempts) {
+			/* retrying */
+			req->callback = callback;
+			http_client_request_resubmit(req);
+			return FALSE;
+		} else {
+			/* release payload early (prevents server/client deadlock in proxy) */
+			if (!sending && req->payload_input != NULL)
+				i_stream_unref(&req->payload_input);
+		}
 	}
 	if (req->payload_wait && req->client->ioloop != NULL)
 		io_loop_stop(req->client->ioloop);
+	return TRUE;
 }
 
 void http_client_request_error_delayed(struct http_client_request **_req)
 {
 	struct http_client_request *req = *_req;
+	bool destroy;
 
 	i_assert(req->state == HTTP_REQUEST_STATE_ABORTED);
 
 	*_req = NULL;
 
 	i_assert(req->delayed_error != NULL && req->delayed_error_status != 0);
-	http_client_request_send_error(req, req->delayed_error_status,
+	destroy = http_client_request_send_error(req, req->delayed_error_status,
 				       req->delayed_error);
 	if (req->queue != NULL)
 		http_client_queue_drop_request(req->queue, req);
-	http_client_request_destroy(&req);
+	if (destroy)
+		http_client_request_destroy(&req);
 }
 
 void http_client_request_error(struct http_client_request **_req,
@@ -1133,8 +1144,8 @@ void http_client_request_error(struct http_client_request **_req,
 		req->delayed_error_status = status;
 		http_client_delay_request_error(req->client, req);
 	} else {
-		http_client_request_send_error(req, status, error);
-		http_client_request_destroy(&req);
+		if (http_client_request_send_error(req, status, error))
+			http_client_request_destroy(&req);
 	}
 	*_req = NULL;
 }

From 29d5559f1934ec12ccad93413502c0873f18b9b5 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Fri, 17 Jun 2016 16:59:15 +0200
Subject: [PATCH 336/450] lib-http: client: Implemented no_auto_retry setting
 that disables all automatic request retries.

This currently only applies to requests sent over a connection that is subsequently lost before a response is received.
Before, such requests were always implicitly resumbitted for a new connection, without the application knowing about it.
By enabling the no_auto_retry client setting, the application is always notified of connection loss through the request's response callback.
As a consequence, requests need to be retried explicitly using the http_client_request_try_retry().
---
 src/lib-http/http-client-connection.c | 18 ++++++++++++++----
 src/lib-http/http-client.c            |  1 +
 src/lib-http/http-client.h            |  3 +++
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/src/lib-http/http-client-connection.c b/src/lib-http/http-client-connection.c
index 9b8a6c3e055..66adf75040d 100644
--- a/src/lib-http/http-client-connection.c
+++ b/src/lib-http/http-client-connection.c
@@ -66,13 +66,19 @@ static void
 http_client_connection_retry_requests(struct http_client_connection *conn,
 	unsigned int status, const char *error)
 {
+	const struct http_client_settings *set = &conn->client->set;
 	struct http_client_request *req, **req_idx;
 
 	if (!array_is_created(&conn->request_wait_list))
 		return;
 
-	http_client_connection_debug(conn,
-		"Retrying pending requests");
+	if (set->no_auto_retry) {
+		http_client_connection_debug(conn,
+			"Aborting pending requests with error");
+	} else {
+		http_client_connection_debug(conn,
+			"Retrying pending requests");
+	}
 
 	array_foreach_modifiable(&conn->request_wait_list, req_idx) {
 		req = *req_idx;
@@ -81,8 +87,12 @@ http_client_connection_retry_requests(struct http_client_connection *conn,
 		if (!http_client_request_unref(req_idx))
 			continue;
 		/* retry the request, which may drop it */
-		if (req->state < HTTP_REQUEST_STATE_FINISHED)
-			http_client_request_retry(req, status, error);
+		if (req->state < HTTP_REQUEST_STATE_FINISHED) {
+			if (set->no_auto_retry)
+				http_client_request_error(&req, status, error);
+			else
+				http_client_request_retry(req, status, error);
+		}
 	}	
 	array_clear(&conn->request_wait_list);
 }
diff --git a/src/lib-http/http-client.c b/src/lib-http/http-client.c
index 321b165ffe7..14814f8047e 100644
--- a/src/lib-http/http-client.c
+++ b/src/lib-http/http-client.c
@@ -137,6 +137,7 @@ struct http_client *http_client_init(const struct http_client_settings *set)
 			HTTP_CLIENT_DEFAULT_BACKOFF_MAX_TIME_MSECS :
 			set->connect_backoff_max_time_msecs;
 	client->set.no_auto_redirect = set->no_auto_redirect;
+	client->set.no_auto_retry = set->no_auto_retry;
 	client->set.no_ssl_tunnel = set->no_ssl_tunnel;
 	client->set.max_redirects = set->max_redirects;
 	client->set.response_hdr_limits = set->response_hdr_limits;
diff --git a/src/lib-http/http-client.h b/src/lib-http/http-client.h
index d482bb1399d..37bdc0220ed 100644
--- a/src/lib-http/http-client.h
+++ b/src/lib-http/http-client.h
@@ -72,6 +72,9 @@ struct http_client_settings {
 	/* don't automatically act upon redirect responses */
 	bool no_auto_redirect;
 
+	/* never automatically retry requests */
+	bool no_auto_retry;
+
 	/* if we use a proxy, delegate SSL negotiation to proxy, rather than
 	   creating a CONNECT tunnel through the proxy for the SSL link */
 	bool no_ssl_tunnel;

From 8c0379c3c2fa966900dd4bcb329108c3fcac7c1f Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Fri, 17 Jun 2016 16:59:37 +0200
Subject: [PATCH 337/450] lib-http: client: Added tests for manual handling of
 connection loss retries to test-http-client-errors.

---
 src/lib-http/test-http-client-errors.c | 39 ++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 5 deletions(-)

diff --git a/src/lib-http/test-http-client-errors.c b/src/lib-http/test-http-client-errors.c
index 4af94e94f9c..579fdf88a5b 100644
--- a/src/lib-http/test-http-client-errors.c
+++ b/src/lib-http/test-http-client-errors.c
@@ -679,17 +679,31 @@ struct _connection_lost_ctx {
 	unsigned int count;
 };
 
+struct _connection_lost_request_ctx {
+	struct _connection_lost_ctx *ctx;
+	struct http_client_request *req;
+};
+
 static void
 test_client_connection_lost_response(
 	const struct http_response *resp,
-	struct _connection_lost_ctx *ctx)
+	struct _connection_lost_request_ctx *rctx)
 {
+	struct _connection_lost_ctx *ctx = rctx->ctx;
+
 	if (debug)
 		i_debug("RESPONSE: %u %s", resp->status, resp->reason);
 
 	test_assert(resp->status == HTTP_CLIENT_REQUEST_ERROR_CONNECTION_LOST);
 	test_assert(resp->reason != NULL && *resp->reason != '\0');
 
+	if (http_client_request_try_retry(rctx->req)) {
+		if (debug)
+			i_debug("retrying");
+		return;
+	}
+	i_free(rctx);
+
 	if (--ctx->count == 0) {
 		i_free(ctx);
 		io_loop_stop(ioloop);
@@ -704,6 +718,7 @@ test_client_connection_lost(const struct http_client_settings *client_set)
 		"server the opportunity to close the connection before the payload "
 		"is finished.";
 	struct _connection_lost_ctx *ctx;
+	struct _connection_lost_request_ctx *rctx;
 	struct http_client_request *hreq;
 	struct istream *input;
 
@@ -714,16 +729,22 @@ test_client_connection_lost(const struct http_client_settings *client_set)
 
 	input = i_stream_create_from_data(payload, sizeof(payload)-1);
 
-	hreq = http_client_request(http_client,
+	rctx = i_new(struct _connection_lost_request_ctx, 1);
+	rctx->ctx = ctx;
+
+	rctx->req = hreq = http_client_request(http_client,
 		"GET", net_ip2addr(&bind_ip), "/connection-lost.txt",
-		test_client_connection_lost_response, ctx);
+		test_client_connection_lost_response, rctx);
 	http_client_request_set_port(hreq, bind_ports[0]);
 	http_client_request_set_payload(hreq, input, FALSE);
 	http_client_request_submit(hreq);
 
-	hreq = http_client_request(http_client,
+	rctx = i_new(struct _connection_lost_request_ctx, 1);
+	rctx->ctx = ctx;
+
+	rctx->req = hreq = http_client_request(http_client,
 		"GET", net_ip2addr(&bind_ip), "/connection-lost2.txt",
-		test_client_connection_lost_response, ctx);
+		test_client_connection_lost_response, rctx);
 	http_client_request_set_port(hreq, bind_ports[0]);
 	http_client_request_submit(hreq);
 
@@ -760,6 +781,14 @@ static void test_connection_lost(void)
 		test_client_connection_lost,
 		test_server_connection_lost, 1);
 	test_end();
+
+	test_begin("connection lost: manual retry");
+	http_client_set.max_attempts = 3;
+	http_client_set.no_auto_retry = TRUE;
+	test_run_client_server(&http_client_set,
+		test_client_connection_lost,
+		test_server_connection_lost, 1);
+	test_end();
 }
 
 /*

From 5ecd956aab342ffd67a9439f6c8be643592fa996 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Sun, 19 Jun 2016 22:35:43 +0200
Subject: [PATCH 338/450] lib-http: server: Added more detailed debugging about
 when idle timeouts are started/stopped.

---
 src/lib-http/http-server-connection.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/lib-http/http-server-connection.c b/src/lib-http/http-server-connection.c
index be3d39055e0..5ecd6b4a1cc 100644
--- a/src/lib-http/http-server-connection.c
+++ b/src/lib-http/http-server-connection.c
@@ -811,15 +811,20 @@ http_server_connection_next_response(struct http_server_connection *conn)
 	req = conn->request_queue_head;
 	if (req == NULL) {
 		/* no requests pending */
+		http_server_connection_debug(conn, "No more requests pending");
 		http_server_connection_timeout_start(conn);
 		return FALSE;
 	}
 	if (req->state < HTTP_SERVER_REQUEST_STATE_READY_TO_RESPOND) {
 		if (req->state == HTTP_SERVER_REQUEST_STATE_PROCESSING) {
 			/* server is causing idle time */
+			http_server_connection_debug(conn,
+				"Not ready to respond: Server is processing");
 			http_server_connection_timeout_stop(conn);
 		} else {
 			/* client is causing idle time */
+			http_server_connection_debug(conn,
+				"Not ready to respond: Waiting for client");
 			http_server_connection_timeout_start(conn);
 		}
 		
@@ -852,6 +857,8 @@ http_server_connection_next_response(struct http_server_connection *conn)
 	i_assert(req->state == HTTP_SERVER_REQUEST_STATE_READY_TO_RESPOND &&
 		req->response != NULL);
 
+	http_server_connection_debug(conn,
+		"Sending response");
 	http_server_connection_timeout_start(conn);
 
 	http_server_request_ref(req);
@@ -940,9 +947,15 @@ int http_server_connection_output(struct http_server_connection *conn)
 				return -1;
 		} else if (conn->io_resp_payload != NULL) {
 			/* server is causing idle time */
+			http_server_connection_debug(conn,
+				"Not ready to continue response: "
+				"Server is producing response");
 			http_server_connection_timeout_stop(conn);
 		} else {
 			/* client is causing idle time */
+			http_server_connection_debug(conn,
+				"Not ready to continue response: "
+				"Waiting for client");
 			http_server_connection_timeout_start(conn);
 		}
 	}

From 83d369656355f44474198df5a87fb3629295e6ca Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Sun, 19 Jun 2016 22:31:18 +0200
Subject: [PATCH 339/450] lib-http: server: Fixed handling of idle timeout when
 request just starts processing on the server.

---
 src/lib-http/http-server-connection.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lib-http/http-server-connection.c b/src/lib-http/http-server-connection.c
index 5ecd6b4a1cc..eba584a5959 100644
--- a/src/lib-http/http-server-connection.c
+++ b/src/lib-http/http-server-connection.c
@@ -234,6 +234,7 @@ static void http_server_payload_destroyed(struct http_server_request *req)
 	case HTTP_SERVER_REQUEST_STATE_PAYLOAD_IN:
 		/* finished reading request */
 		req->state = HTTP_SERVER_REQUEST_STATE_PROCESSING;
+		http_server_connection_timeout_stop(conn);
 		if (req->response != NULL && req->response->submitted)
 			http_server_request_submit_response(req);
 		break;

From d8570b292afd4ee542a0b91ef69914865f0d0931 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Mon, 20 Jun 2016 01:15:18 +0200
Subject: [PATCH 340/450] lib-http: server: Wrap request payload in a timeout
 input stream.

The server cannot reset the timeout properly while the payload is being read by the application.
The HTTP client solved this same problem by identical means.
---
 src/lib-http/http-server-connection.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/lib-http/http-server-connection.c b/src/lib-http/http-server-connection.c
index eba584a5959..5425ff8a90e 100644
--- a/src/lib-http/http-server-connection.c
+++ b/src/lib-http/http-server-connection.c
@@ -6,6 +6,7 @@
 #include "str.h"
 #include "ioloop.h"
 #include "istream.h"
+#include "istream-timeout.h"
 #include "ostream.h"
 #include "connection.h"
 #include "iostream-rawlog.h"
@@ -302,6 +303,7 @@ static bool
 http_server_connection_handle_request(struct http_server_connection *conn,
 	struct http_server_request *req)
 {
+	const struct http_server_settings *set = &conn->server->set;
 	struct istream *payload;
 
 	i_assert(!conn->in_req_callback);
@@ -318,7 +320,11 @@ http_server_connection_handle_request(struct http_server_connection *conn,
 		/* wrap the stream to capture the destroy event without destroying the
 		   actual payload stream. */
 		conn->incoming_payload = req->req.payload =
-			i_stream_create_limit(req->req.payload, (uoff_t)-1);
+			i_stream_create_timeout(req->req.payload,
+				set->max_client_idle_time_msecs);
+		/* we've received the request itself, and we can't reset the
+		   timeout during the payload reading. */
+		http_server_connection_timeout_stop(conn);
 	} else {
 		conn->incoming_payload = req->req.payload =
 			i_stream_create_from_data("", 0);

From 189b586eb455a484ba2153ba2512db472e8c6032 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Mon, 20 Jun 2016 01:20:33 +0200
Subject: [PATCH 341/450] lib-http: server: Make sure provided connection FDs
 are non-blocking.

---
 src/lib-http/http-server-connection.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/lib-http/http-server-connection.c b/src/lib-http/http-server-connection.c
index 5425ff8a90e..daf3e226382 100644
--- a/src/lib-http/http-server-connection.c
+++ b/src/lib-http/http-server-connection.c
@@ -1025,6 +1025,10 @@ http_server_connection_create(struct http_server *server,
 	conn->callbacks = callbacks;
 	conn->context = context;
 
+	net_set_nonblock(fd_in, TRUE);
+	if (fd_in != fd_out)
+		net_set_nonblock(fd_out, TRUE);
+
 	/* get a name for this connection */
 	if (fd_in != fd_out || net_getpeername(fd_in, &addr, &port) < 0) {
 		name = t_strdup_printf("[%u]", id);

From 183ca5c9e1ab5ee712ef6700021d3c1d18ab7f8c Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Mon, 20 Jun 2016 01:25:02 +0200
Subject: [PATCH 342/450] lib-http: server: Added settings to configure the
 connection's socket kernel buffer sizes.

This is mainly useful for use in the lib-http test suite.
---
 src/lib-http/http-server-connection.c | 14 ++++++++++++++
 src/lib-http/http-server.c            |  2 ++
 src/lib-http/http-server.h            |  6 ++++++
 3 files changed, 22 insertions(+)

diff --git a/src/lib-http/http-server-connection.c b/src/lib-http/http-server-connection.c
index daf3e226382..0b7b8c84afe 100644
--- a/src/lib-http/http-server-connection.c
+++ b/src/lib-http/http-server-connection.c
@@ -1011,6 +1011,7 @@ http_server_connection_create(struct http_server *server,
 	int fd_in, int fd_out, bool ssl,
 	const struct http_server_callbacks *callbacks, void *context)
 {
+	const struct http_server_settings *set = &server->set;
 	struct http_server_connection *conn;
 	static unsigned int id = 0;
 	struct ip_addr addr;
@@ -1029,6 +1030,19 @@ http_server_connection_create(struct http_server *server,
 	if (fd_in != fd_out)
 		net_set_nonblock(fd_out, TRUE);
 
+	if (set->socket_send_buffer_size > 0) {
+		if (net_set_send_buffer_size(fd_out,
+			set->socket_send_buffer_size) < 0)
+			i_error("net_set_send_buffer_size(%"PRIuSIZE_T") failed: %m",
+				set->socket_send_buffer_size);
+	}
+	if (set->socket_recv_buffer_size > 0) {
+		if (net_set_recv_buffer_size(fd_in,
+			set->socket_recv_buffer_size) < 0)
+			i_error("net_set_recv_buffer_size(%"PRIuSIZE_T") failed: %m",
+				set->socket_recv_buffer_size);
+	}
+
 	/* get a name for this connection */
 	if (fd_in != fd_out || net_getpeername(fd_in, &addr, &port) < 0) {
 		name = t_strdup_printf("[%u]", id);
diff --git a/src/lib-http/http-server.c b/src/lib-http/http-server.c
index 7f9f28df4fc..8a3d4a99753 100644
--- a/src/lib-http/http-server.c
+++ b/src/lib-http/http-server.c
@@ -34,6 +34,8 @@ struct http_server *http_server_init(const struct http_server_settings *set)
 	server->set.max_pipelined_requests =
 		(set->max_pipelined_requests > 0 ? set->max_pipelined_requests : 1);
 	server->set.request_limits = set->request_limits;
+	server->set.socket_send_buffer_size = set->socket_send_buffer_size;
+	server->set.socket_recv_buffer_size = set->socket_recv_buffer_size;
 	server->set.debug = set->debug;
 
 	server->conn_list = http_server_connection_list_init();
diff --git a/src/lib-http/http-server.h b/src/lib-http/http-server.h
index b90a3967312..5d87f0fcdd2 100644
--- a/src/lib-http/http-server.h
+++ b/src/lib-http/http-server.h
@@ -24,6 +24,12 @@ struct http_server_settings {
 	/* request limits */
 	struct http_request_limits request_limits;
 
+	/* the kernel send/receive buffer sizes used for the connection sockets.
+	   Configuring this is mainly useful for the test suite. The kernel
+	   defaults are used when these settings are 0. */
+	size_t socket_send_buffer_size;
+	size_t socket_recv_buffer_size;
+
 	bool debug;
 };
 

From 3da0c5295afd0c4fbe0233687a58b2d95d6131f4 Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Mon, 20 Jun 2016 02:41:37 +0200
Subject: [PATCH 343/450] lib-http: Restructured Makefile.am to have less
 duplicated lines for the test suite.

---
 src/lib-http/Makefile.am | 55 +++++++++++-----------------------------
 1 file changed, 15 insertions(+), 40 deletions(-)

diff --git a/src/lib-http/Makefile.am b/src/lib-http/Makefile.am
index 73941e8f4ea..a1c972d8ede 100644
--- a/src/lib-http/Makefile.am
+++ b/src/lib-http/Makefile.am
@@ -132,16 +132,14 @@ test_http_request_parser_LDADD = \
 	$(test_libs)
 test_http_request_parser_DEPENDENCIES = $(test_deps)
 
-test_http_payload_SOURCES = test-http-payload.c
-test_http_payload_LDFLAGS = -export-dynamic
-test_http_payload_LDADD = \
+test_http_libs = \
 	libhttp.la \
 	../lib-dns/libdns.la \
 	../lib-ssl-iostream/libssl_iostream.la \
 	../lib-master/libmaster.la \
 	../lib-settings/libsettings.la \
 	$(test_libs)
-test_http_payload_DEPENDENCIES = \
+test_http_deps = \
 	libhttp.la \
 	../lib-dns/libdns.la \
 	../lib-ssl-iostream/libssl_iostream.la \
@@ -149,56 +147,33 @@ test_http_payload_DEPENDENCIES = \
 	../lib-settings/libsettings.la \
 	$(test_deps)
 
+test_http_payload_SOURCES = test-http-payload.c
+test_http_payload_LDFLAGS = -export-dynamic
+test_http_payload_LDADD = \
+	$(test_http_libs)
+test_http_payload_DEPENDENCIES = \
+	$(test_http_deps)
+
 test_http_client_SOURCES = test-http-client.c
 test_http_client_LDFLAGS = -export-dynamic
 test_http_client_LDADD = \
-	libhttp.la \
-	../lib-dns/libdns.la \
-	../lib-ssl-iostream/libssl_iostream.la \
-	../lib-master/libmaster.la \
-	../lib-settings/libsettings.la \
-	$(test_libs)
+	$(test_http_libs)
 test_http_client_DEPENDENCIES = \
-	libhttp.la \
-	../lib-dns/libdns.la \
-	../lib-ssl-iostream/libssl_iostream.la \
-	../lib-master/libmaster.la \
-	../lib-settings/libsettings.la \
-	$(test_deps)
+	$(test_http_deps)
 
 test_http_client_errors_SOURCES = test-http-client-errors.c
 test_http_client_errors_LDFLAGS = -export-dynamic
 test_http_client_errors_LDADD = \
-	libhttp.la \
-	../lib-dns/libdns.la \
-	../lib-ssl-iostream/libssl_iostream.la \
-	../lib-master/libmaster.la \
-	../lib-settings/libsettings.la \
-	$(test_libs)
+	$(test_http_libs)
 test_http_client_errors_DEPENDENCIES = \
-	libhttp.la \
-	../lib-dns/libdns.la \
-	../lib-ssl-iostream/libssl_iostream.la \
-	../lib-master/libmaster.la \
-	../lib-settings/libsettings.la \
-	$(test_deps)
+	$(test_http_deps)
 
 test_http_server_SOURCES = test-http-server.c
 test_http_server_LDFLAGS = -export-dynamic
 test_http_server_LDADD = \
-	libhttp.la \
-	../lib-dns/libdns.la \
-	../lib-ssl-iostream/libssl_iostream.la \
-	../lib-master/libmaster.la \
-	../lib-settings/libsettings.la \
-	$(test_libs)
+	$(test_http_libs)
 test_http_server_DEPENDENCIES = \
-	libhttp.la \
-	../lib-dns/libdns.la \
-	../lib-ssl-iostream/libssl_iostream.la \
-	../lib-master/libmaster.la \
-	../lib-settings/libsettings.la \
-	$(test_deps)
+	$(test_http_deps)
 
 check: check-am check-test
 check-test: all-am

From f292589f4b85e02d97d974dfe34324c6c0bb9d9f Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@dovecot.fi>
Date: Sun, 19 Jun 2016 21:31:10 +0200
Subject: [PATCH 344/450] lib-http: server: Created test program that tests
 error conditions.

Currently it is very limited, but it is due to be extended soon towards testing most common error conditions.
---
 src/lib-http/Makefile.am               |  10 +-
 src/lib-http/test-http-server-errors.c | 754 +++++++++++++++++++++++++
 2 files changed, 763 insertions(+), 1 deletion(-)
 create mode 100644 src/lib-http/test-http-server-errors.c

diff --git a/src/lib-http/Makefile.am b/src/lib-http/Makefile.am
index a1c972d8ede..967002e7b81 100644
--- a/src/lib-http/Makefile.am
+++ b/src/lib-http/Makefile.am
@@ -65,7 +65,8 @@ test_nocheck_programs = \
 	test-http-payload \
 	test-http-client \
 	test-http-client-errors \
-	test-http-server
+	test-http-server \
+	test-http-server-errors
 
 noinst_PROGRAMS = $(test_programs) $(test_nocheck_programs)
 
@@ -175,6 +176,13 @@ test_http_server_LDADD = \
 test_http_server_DEPENDENCIES = \
 	$(test_http_deps)
 
+test_http_server_errors_SOURCES = test-http-server-errors.c
+test_http_server_errors_LDFLAGS = -export-dynamic
+test_http_server_errors_LDADD = \
+	$(test_http_libs)
+test_http_server_errors_DEPENDENCIES = \
+	$(test_http_deps)
+
 check: check-am check-test
 check-test: all-am
 	for bin in $(test_programs); do \
diff --git a/src/lib-http/test-http-server-errors.c b/src/lib-http/test-http-server-errors.c
new file mode 100644
index 00000000000..9c432ce3c60
--- /dev/null
+++ b/src/lib-http/test-http-server-errors.c
@@ -0,0 +1,754 @@
+/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "hostpid.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "ostream.h"
+#include "time-util.h"
+#include "connection.h"
+#include "test-common.h"
+#include "http-url.h"
+#include "http-request.h"
+#include "http-server.h"
+
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <signal.h>
+#include <unistd.h>
+
+#define SERVER_MAX_TIMEOUT_MSECS 10*1000
+
+/*
+ * Types
+ */
+
+struct client_connection {
+	struct connection conn;
+
+	pool_t pool;
+};
+
+typedef void (*test_server_init_t)
+	(const struct http_server_settings *server_set);
+typedef void (*test_client_init_t)(unsigned int index);
+
+/*
+ * State
+ */
+
+/* common */
+static struct ip_addr bind_ip;
+static in_port_t bind_port = 0;
+static struct ioloop *ioloop;
+static bool debug = FALSE;
+
+/* server */
+static struct http_server *http_server = NULL;
+static struct io *io_listen;
+static int fd_listen = -1;
+static void (*test_server_request)(struct http_server_request *req);
+
+/* client */
+static pid_t *client_pids = NULL;
+static struct connection_list *client_conn_list;
+static unsigned int client_pids_count = 0;
+static unsigned int client_index;
+static void (*test_client_connected)(struct client_connection *conn);
+static void (*test_client_input)(struct client_connection *conn);
+
+/*
+ * Forward declarations
+ */
+
+/* server */
+static void
+test_server_defaults(struct http_server_settings *http_set);
+static void
+test_server_run(const struct http_server_settings *http_set);
+
+/* client */
+static void test_client_run(unsigned int index);
+
+/* test*/
+static void test_run_client_server(
+	const struct http_server_settings *server_set,
+	test_server_init_t server_test,
+	test_client_init_t client_test,
+	unsigned int client_tests_count)
+	ATTR_NULL(3);
+
+/*
+ * Slow request
+ */
+
+/* client */
+
+static void
+test_slow_request_input(struct client_connection *conn ATTR_UNUSED)
+{
+	/* do nothing */
+}
+
+static void
+test_slow_request_connected(struct client_connection *conn)
+{
+	(void)o_stream_send_str(conn->conn.output,
+		"GET / HTTP/1.1\r\n"
+		"Host: example.com\r\n"
+		"\r\n");
+}
+
+static void test_client_slow_request(unsigned int index)
+{
+	test_client_input = test_slow_request_input;
+	test_client_connected = test_slow_request_connected;
+	test_client_run(index);
+}
+
+/* server */
+
+struct _slow_request {
+	struct http_server_request *req;
+	struct timeout *to_delay;
+	unsigned int serviced:1;
+};
+
+static void
+test_server_slow_request_destroyed(void *context)
+{
+	struct _slow_request *ctx = (struct _slow_request *)context;
+	test_assert(ctx->serviced);
+	if (ctx->to_delay != NULL)
+		timeout_remove(&ctx->to_delay);
+	i_free(ctx);
+	io_loop_stop(ioloop);
+}
+
+static void
+test_server_slow_request_delayed(struct _slow_request *ctx)
+{
+	struct http_server_response *resp;
+	struct http_server_request *req = ctx->req;
+
+	resp = http_server_response_create(req, 200, "OK");
+	http_server_response_submit(resp);
+	ctx->serviced = TRUE;
+
+	http_server_request_unref(&req);
+}
+
+static void
+test_server_slow_request_request(
+	struct http_server_request *req)
+{
+	const struct http_request *hreq =
+		http_server_request_get(req);
+	struct _slow_request *ctx;
+
+	if (debug) {
+		i_debug("REQUEST: %s %s HTTP/%u.%u",
+			hreq->method, hreq->target_raw,
+			hreq->version_major, hreq->version_minor);
+	}
+
+	ctx = i_new(struct _slow_request, 1);
+	ctx->req = req;
+
+	http_server_request_set_destroy_callback(req,
+		test_server_slow_request_destroyed, ctx);
+
+	http_server_request_ref(req);
+	ctx->to_delay = timeout_add
+		(4000, test_server_slow_request_delayed, ctx);
+}
+
+static void test_server_slow_request
+(const struct http_server_settings *server_set)
+{
+	test_server_request = test_server_slow_request_request;
+	test_server_run(server_set);
+}
+
+/* test */
+
+static void test_slow_request(void)
+{
+	struct http_server_settings http_server_set;
+
+	test_server_defaults(&http_server_set);
+	http_server_set.max_client_idle_time_msecs = 1000;
+
+	test_begin("slow request");
+	test_run_client_server(&http_server_set,
+		test_server_slow_request,
+		test_client_slow_request, 1);
+	test_end();
+}
+
+/*
+ * Hanging request payload
+ */
+
+/* client */
+
+static void
+test_hanging_request_payload_connected(struct client_connection *conn)
+{
+	(void)o_stream_send_str(conn->conn.output,
+		"GET / HTTP/1.1\r\n"
+		"Host: example.com\r\n"
+		"Content-Length: 1000\r\n"
+		"\r\n"
+		"To be continued... or not");
+}
+
+static void test_client_hanging_request_payload(unsigned int index)
+{
+	test_client_connected = test_hanging_request_payload_connected;
+	test_client_run(index);
+}
+
+/* server */
+
+struct _hanging_request_payload {
+	struct http_server_request *req;
+	struct istream *payload_input;
+	struct io *io;
+	unsigned int serviced:1;
+};
+
+static void
+test_server_hanging_request_payload_destroyed(void *context)
+{
+	struct _hanging_request_payload *ctx =
+		(struct _hanging_request_payload *)context;
+	test_assert(!ctx->serviced);
+	if (ctx->io != NULL)
+		io_remove(&ctx->io);
+	i_free(ctx);
+	io_loop_stop(ioloop);
+}
+
+static void
+test_server_hanging_request_payload_input(struct _hanging_request_payload *ctx)
+{
+	struct http_server_response *resp;
+	struct http_server_request *req = ctx->req;
+	const unsigned char *data;
+	size_t size;
+	int ret;
+
+	if (debug)
+		i_debug("test server: got more payload");
+
+	while ((ret=i_stream_read_data
+		(ctx->payload_input, &data, &size, 0)) > 0) {
+		i_stream_skip(ctx->payload_input, size);
+	}
+
+	if (ret == 0)
+		return;
+	if (ret < 0) {
+		if (debug) {
+			i_debug("test server: failed to read payload: %s",
+				i_stream_get_error(ctx->payload_input));
+		}
+		i_stream_unref(&ctx->payload_input);
+		io_remove(&ctx->io);
+		http_server_request_fail_close(req,
+			400, "Bad request");
+		http_server_request_unref(&req);
+		return;
+	}
+		
+	resp = http_server_response_create(req, 200, "OK");
+	http_server_response_submit(resp);
+	ctx->serviced = TRUE;
+
+	i_stream_unref(&ctx->payload_input);
+	http_server_request_unref(&req);
+}
+
+static void
+test_server_hanging_request_payload_request(
+	struct http_server_request *req)
+{
+	const struct http_request *hreq =
+		http_server_request_get(req);
+	struct _hanging_request_payload *ctx;
+
+	if (debug) {
+		i_debug("REQUEST: %s %s HTTP/%u.%u",
+			hreq->method, hreq->target_raw,
+			hreq->version_major, hreq->version_minor);
+	}
+
+	ctx = i_new(struct _hanging_request_payload, 1);
+	ctx->req = req;
+
+	http_server_request_set_destroy_callback(req,
+		test_server_hanging_request_payload_destroyed, ctx);
+
+	ctx->payload_input =
+		http_server_request_get_payload_input(req, FALSE);
+
+	http_server_request_ref(req);
+	ctx->io = io_add_istream(ctx->payload_input,
+		test_server_hanging_request_payload_input, ctx);
+	test_server_hanging_request_payload_input(ctx);
+}
+
+static void test_server_hanging_request_payload
+(const struct http_server_settings *server_set)
+{
+	test_server_request = test_server_hanging_request_payload_request;
+	test_server_run(server_set);
+}
+
+/* test */
+
+static void test_hanging_request_payload(void)
+{
+	struct http_server_settings http_server_set;
+
+	test_server_defaults(&http_server_set);
+	http_server_set.max_client_idle_time_msecs = 1000;
+
+	test_begin("hanging request payload");
+	test_run_client_server(&http_server_set,
+		test_server_hanging_request_payload,
+		test_client_hanging_request_payload, 1);
+	test_end();
+}
+
+/*
+ * Hanging response payload
+ */
+
+/* client */
+
+static void
+test_hanging_response_payload_connected(struct client_connection *conn)
+{
+	(void)o_stream_send_str(conn->conn.output,
+		"GET / HTTP/1.1\r\n"
+		"Host: example.com\r\n"
+		"Content-Length: 18\r\n"
+		"\r\n"
+		"Complete payload\r\n");
+}
+
+static void test_client_hanging_response_payload(unsigned int index)
+{
+	test_client_connected = test_hanging_response_payload_connected;
+	test_client_run(index);
+}
+
+/* server */
+
+struct _hanging_response_payload {
+	struct http_server_request *req;
+	struct istream *payload_input;
+	struct io *io;
+	unsigned int serviced:1;
+};
+
+static void
+test_server_hanging_response_payload_destroyed(void *context)
+{
+	struct _hanging_response_payload *ctx =
+		(struct _hanging_response_payload *)context;
+	test_assert(!ctx->serviced);
+	if (ctx->io != NULL)
+		io_remove(&ctx->io);
+	i_free(ctx);
+	io_loop_stop(ioloop);
+}
+
+static void
+test_server_hanging_response_payload_request(
+	struct http_server_request *req)
+{
+	const struct http_request *hreq =
+		http_server_request_get(req);
+	struct http_server_response *resp;
+	struct _hanging_response_payload *ctx;
+	string_t *payload;
+	unsigned int i;
+
+	if (debug) {
+		i_debug("REQUEST: %s %s HTTP/%u.%u",
+			hreq->method, hreq->target_raw,
+			hreq->version_major, hreq->version_minor);
+	}
+
+	ctx = i_new(struct _hanging_response_payload, 1);
+	ctx->req = req;
+
+	http_server_request_set_destroy_callback(req,
+		test_server_hanging_response_payload_destroyed, ctx);
+
+	resp = http_server_response_create(req, 200, "OK");
+	T_BEGIN {
+		payload = t_str_new(204800);
+		for (i = 0; i < 3200; i++) {
+			str_append(payload,
+				"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n"
+				"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\r\n");
+		}
+
+		http_server_response_set_payload_data
+			(resp, str_data(payload), str_len(payload));
+	} T_END;
+	http_server_response_submit(resp);
+}
+
+static void test_server_hanging_response_payload
+(const struct http_server_settings *server_set)
+{
+	test_server_request = test_server_hanging_response_payload_request;
+	test_server_run(server_set);
+}
+
+/* test */
+
+static void test_hanging_response_payload(void)
+{
+	struct http_server_settings http_server_set;
+
+	test_server_defaults(&http_server_set);
+	http_server_set.socket_send_buffer_size = 4096;
+	http_server_set.max_client_idle_time_msecs = 1000;
+
+	test_begin("hanging response payload");
+	test_run_client_server(&http_server_set,
+		test_server_hanging_response_payload,
+		test_client_hanging_response_payload, 1);
+	test_end();
+}
+
+/*
+ * All tests
+ */
+
+static void (*test_functions[])(void) = {
+	test_slow_request,
+	test_hanging_request_payload,
+	test_hanging_response_payload,
+	NULL
+};
+
+/*
+ * Test client
+ */
+
+/* client connection */
+
+static void
+client_connection_input(struct connection *_conn)
+{
+	struct client_connection *conn = (struct client_connection *)_conn;
+	
+	if (test_client_input != NULL)
+		test_client_input(conn);
+}
+
+static void
+client_connection_connected(struct connection *_conn, bool success)
+{
+	struct client_connection *conn = (struct client_connection *)_conn;
+
+	if (success && test_client_connected != NULL)
+		test_client_connected(conn);
+}
+
+static void
+client_connection_init(const struct ip_addr *ip, in_port_t port)
+{
+	struct client_connection *conn;
+	pool_t pool;
+
+	pool = pool_alloconly_create("client connection", 256);
+	conn = p_new(pool, struct client_connection, 1);
+	conn->pool = pool;
+
+	connection_init_client_ip(client_conn_list,
+		&conn->conn, ip, port);
+	(void)connection_client_connect(&conn->conn);
+}
+
+static void
+server_connection_deinit(struct client_connection **_conn)
+{
+	struct client_connection *conn = *_conn;
+
+	*_conn = NULL;
+
+	connection_deinit(&conn->conn);
+	pool_unref(&conn->pool);
+}
+
+static void
+client_connection_destroy(struct connection *_conn)
+{
+	struct client_connection *conn =
+		(struct client_connection *)_conn;
+
+	server_connection_deinit(&conn);
+}
+
+/* */
+
+static struct connection_settings client_connection_set = {
+	.input_max_size = (size_t)-1,
+	.output_max_size = (size_t)-1,
+	.client = TRUE
+};
+
+static const struct connection_vfuncs client_connection_vfuncs = {
+	.destroy = client_connection_destroy,
+	.client_connected = client_connection_connected,
+	.input = client_connection_input
+};
+
+static void test_client_run(unsigned int index)
+{
+	client_index = index;
+
+	if (debug)
+		i_debug("client connecting to %u", bind_port);
+
+	client_conn_list = connection_list_init
+		(&client_connection_set, &client_connection_vfuncs);
+
+	client_connection_init(&bind_ip, bind_port);
+
+	io_loop_run(ioloop);
+
+	/* close server socket */
+	io_remove(&io_listen);
+
+	connection_list_deinit(&client_conn_list);
+}
+
+/*
+ * Test server
+ */
+
+static void
+test_server_defaults(struct http_server_settings *http_set)
+{
+	/* server settings */
+	memset(http_set, 0, sizeof(*http_set));
+	http_set->max_client_idle_time_msecs = 5*1000;
+	http_set->max_pipelined_requests = 1;
+	http_set->debug = debug;
+}
+
+/* client connection */
+
+static void
+server_handle_request(void *context ATTR_UNUSED,
+	struct http_server_request *req)
+{
+	test_server_request(req);
+}
+
+struct http_server_callbacks http_server_callbacks = {
+	.handle_request = server_handle_request
+};
+
+static void
+server_connection_accept(void *context ATTR_UNUSED)
+{
+	int fd;
+
+	/* accept new client */
+	fd = net_accept(fd_listen, NULL, NULL);
+	if (fd == -1)
+		return;
+	if (fd == -2) {
+		i_fatal("test server: accept() failed: %m");
+	}
+
+	(void)http_server_connection_create(http_server, fd, fd, FALSE,
+		&http_server_callbacks, NULL);
+}
+
+/* */
+
+static void
+test_server_timeout(void *context ATTR_UNUSED)
+{
+	i_fatal("Server timed out");
+	io_loop_stop(ioloop);
+}
+
+static void
+test_server_run(const struct http_server_settings *http_set)
+{
+	struct timeout *to;
+
+	to = timeout_add(SERVER_MAX_TIMEOUT_MSECS,
+		test_server_timeout, NULL);
+
+	/* open server socket */
+	io_listen = io_add(fd_listen,
+		IO_READ, server_connection_accept, (void *)NULL);
+
+	http_server = http_server_init(http_set);
+
+	io_loop_run(ioloop);
+
+	/* close server socket */
+	io_remove(&io_listen);
+	timeout_remove(&to);
+
+	http_server_deinit(&http_server);
+}
+
+/*
+ * Tests
+ */
+
+static int test_open_server_fd(void)
+{
+	int fd = net_listen(&bind_ip, &bind_port, 128);
+	if (debug)
+		i_debug("server listening on %u", bind_port);
+	if (fd == -1) {
+		i_fatal("listen(%s:%u) failed: %m",
+			net_ip2addr(&bind_ip), bind_port);
+	}
+	return fd;
+}
+
+static void test_clients_kill_all(void)
+{
+	unsigned int i;
+
+	if (client_pids_count > 0) {
+		for (i = 0; i < client_pids_count; i++) {
+			if (client_pids[i] != (pid_t)-1) {
+				(void)kill(client_pids[i], SIGKILL);
+				(void)waitpid(client_pids[i], NULL, 0);
+				client_pids[i] = -1;
+			}
+		}
+	}
+	client_pids_count = 0;
+}
+
+static void test_run_client_server(
+	const struct http_server_settings *server_set,
+	test_server_init_t server_test,
+	test_client_init_t client_test,
+	unsigned int client_tests_count)
+{
+	unsigned int i;
+
+	client_pids = NULL;
+	client_pids_count = 0;
+
+	fd_listen = test_open_server_fd();
+
+	if (client_tests_count > 0) {
+		client_pids = i_new(pid_t, client_tests_count);
+		for (i = 0; i < client_tests_count; i++)
+			client_pids[i] = (pid_t)-1;
+		client_pids_count = client_tests_count;
+
+		for (i = 0; i < client_tests_count; i++) {
+			if ((client_pids[i] = fork()) == (pid_t)-1)
+				i_fatal("fork() failed: %m");
+			if (client_pids[i] == 0) {
+				client_pids[i] = (pid_t)-1;
+				client_pids_count = 0;
+				hostpid_init();
+				if (debug)
+					i_debug("client[%d]: PID=%s", i+1, my_pid);
+				/* child: client */
+				usleep(100000); /* wait a little for server setup */
+				i_close_fd(&fd_listen);
+				ioloop = io_loop_create();
+				client_test(i);
+				io_loop_destroy(&ioloop);
+				i_free(client_pids);
+				/* wait for it to be killed; this way, valgrind will not
+				   object to this process going away inelegantly. */
+				sleep(60);
+				exit(1);
+			}
+		}
+		if (debug)
+			i_debug("server: PID=%s", my_pid);
+	}
+
+	/* parent: server */
+
+	ioloop = io_loop_create();
+	server_test(server_set);
+	io_loop_destroy(&ioloop);
+
+	i_close_fd(&fd_listen);
+
+	test_clients_kill_all();
+	i_free(client_pids);
+}
+
+/*
+ * Main
+ */
+
+volatile sig_atomic_t terminating = 0;
+
+static void
+test_signal_handler(int signo)
+{
+	if (terminating != 0)
+		raise(signo);
+	terminating = 1;
+
+	/* make sure we don't leave any pesky children alive */
+	test_clients_kill_all();
+
+	(void)signal(signo, SIG_DFL);
+	raise(signo);
+}
+
+static void test_atexit(void)
+{
+	test_clients_kill_all();
+}
+
+int main(int argc, char *argv[])
+{
+	int c;
+
+	atexit(test_atexit);
+	(void)signal(SIGCHLD, SIG_IGN);
+	(void)signal(SIGTERM, test_signal_handler);
+	(void)signal(SIGQUIT, test_signal_handler);
+	(void)signal(SIGINT, test_signal_handler);
+	(void)signal(SIGSEGV, test_signal_handler);
+	(void)signal(SIGABRT, test_signal_handler);
+
+  while ((c = getopt(argc, argv, "D")) > 0) {
+		switch (c) {
+		case 'D':
+			debug = TRUE;
+			break;
+		default:
+			i_fatal("Usage: %s [-D]", argv[0]);
+		}
+  }
+
+	/* listen on localhost */
+	memset(&bind_ip, 0, sizeof(bind_ip));
+	bind_ip.family = AF_INET;
+	bind_ip.u.ip4.s_addr = htonl(INADDR_LOOPBACK);	
+
+	test_run(test_functions);
+}

From 8fbcc7a11588d8ac1dbc611d888b21fb85d21768 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 13:05:37 +0300
Subject: [PATCH 345/450] lib-dcrypt: Fix to backend module loading.

Especially don't die if we fail to load the module.
---
 src/lib-dcrypt/dcrypt.c | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt.c b/src/lib-dcrypt/dcrypt.c
index 4312848cdb2..fa5b4275156 100644
--- a/src/lib-dcrypt/dcrypt.c
+++ b/src/lib-dcrypt/dcrypt.c
@@ -9,6 +9,7 @@ static struct dcrypt_vfs *dcrypt_vfs = NULL;
 bool dcrypt_initialize(const char *backend, const char **error_r)
 {
 	struct module_dir_load_settings mod_set;
+	const char *error;
 
 	if (dcrypt_vfs != NULL) {
 		return TRUE;
@@ -19,19 +20,14 @@ bool dcrypt_initialize(const char *backend, const char **error_r)
 
 	memset(&mod_set, 0, sizeof(mod_set));
 	mod_set.abi_version = DOVECOT_ABI_VERSION;
-	mod_set.require_init_funcs = 1;
-	dcrypt_module = module_dir_load(DCRYPT_MODULE_DIR, implementation, &mod_set);
-	if (dcrypt_module == NULL) {
+	mod_set.require_init_funcs = TRUE;
+	if (module_dir_try_load_missing(&dcrypt_module, DCRYPT_MODULE_DIR,
+					implementation, &mod_set, &error) < 0) {
 		if (error_r != NULL)
-			*error_r = "No such module";
+			*error_r = error;
 		return FALSE;
 	}
-	if (dcrypt_module->init == NULL) {
-		if (error_r != NULL)
-			*error_r = "Module missing init/deinit";
-		return FALSE;
-	}
-	dcrypt_module->init(dcrypt_module);
+	module_dir_init(dcrypt_module);
 	i_assert(dcrypt_vfs != NULL);
 	/* Destroy SSL module after(most of) the others. Especially lib-fs
 	   backends may still want to access SSL module in their own
@@ -42,10 +38,7 @@ bool dcrypt_initialize(const char *backend, const char **error_r)
 
 void dcrypt_deinitialize(void)
 {
-	if (dcrypt_module != NULL) {
-		dcrypt_module->deinit();
-		module_dir_unload(&dcrypt_module);
-	}
+	module_dir_unload(&dcrypt_module);
 }
 
 void dcrypt_set_vfs(struct dcrypt_vfs *vfs)

From bff052bd29dbf7175ee6cd14bd14bcea1900b869 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 12:58:08 +0300
Subject: [PATCH 346/450] lib-dcrypt, lib-ssl-iostream: Share OpenSSL
 init/deinit code.

---
 src/Makefile.am                               |  2 +-
 src/lib-dcrypt/Makefile.am                    |  6 +-
 src/lib-dcrypt/dcrypt-openssl.c               |  8 +-
 src/lib-ssl-iostream/Makefile.am              | 10 ++-
 src/lib-ssl-iostream/dovecot-openssl-common.c | 81 +++++++++++++++++++
 src/lib-ssl-iostream/dovecot-openssl-common.h | 16 ++++
 .../iostream-openssl-context.c                | 38 +++------
 7 files changed, 124 insertions(+), 37 deletions(-)
 create mode 100644 src/lib-ssl-iostream/dovecot-openssl-common.c
 create mode 100644 src/lib-ssl-iostream/dovecot-openssl-common.h

diff --git a/src/Makefile.am b/src/Makefile.am
index 086b7c8fb5b..dd76173d8a4 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -9,11 +9,11 @@ LIBDOVECOT_SUBDIRS = \
 	lib-auth \
 	lib-master \
 	lib-charset \
+	lib-ssl-iostream \
 	lib-dcrypt \
 	lib-dns \
 	lib-dict \
 	lib-sasl \
-	lib-ssl-iostream \
 	lib-stats \
 	lib-http \
 	lib-fs \
diff --git a/src/lib-dcrypt/Makefile.am b/src/lib-dcrypt/Makefile.am
index c7618f0dc5c..af25c10b53a 100644
--- a/src/lib-dcrypt/Makefile.am
+++ b/src/lib-dcrypt/Makefile.am
@@ -3,7 +3,8 @@ pkglib_LTLIBRARIES =
 
 AM_CPPFLAGS = \
 	-I$(top_srcdir)/src/lib \
-	-I$(top_srcdir)/src/lib-test
+	-I$(top_srcdir)/src/lib-test \
+	-I$(top_srcdir)/src/lib-ssl-iostream
 
 libdcrypt_la_SOURCES = \
 	dcrypt.c \
@@ -16,7 +17,8 @@ libdcrypt_la_CFLAGS = $(AM_CPPFLAGS) \
 if BUILD_OPENSSL
 pkglib_LTLIBRARIES += libdcrypt_openssl.la
 libdcrypt_openssl_la_SOURCES = dcrypt-openssl.c dcrypt.c
-libdcrypt_openssl_la_LDFLAGS = -module -avoid-version -shared $(SSL_LIBS) ../lib/liblib.la
+libdcrypt_openssl_la_LDFLAGS = -module -avoid-version -shared ../lib-ssl-iostream/libdovecot_openssl_common.la ../lib/liblib.la
+libdcrypt_openssl_la_DEPENDENCIES = ../lib-ssl-iostream/libdovecot_openssl_common.la ../lib/liblib.la
 libdcrypt_openssl_la_CFLAGS = $(AM_CPPFLAGS) \
 	-DDCRYPT_MODULE_DIR=\"$(pkglibdir)\"
 	$(SSL_CFLAGS)
diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index af230ee0282..e7cfbb5374d 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -6,6 +6,7 @@
 #include "randgen.h"
 #include "array.h"
 #include "module-dir.h"
+#include "dovecot-openssl-common.h"
 #include <openssl/evp.h>
 #include <openssl/sha.h>
 #include <openssl/err.h>
@@ -2059,14 +2060,11 @@ static struct dcrypt_vfs dcrypt_openssl_vfs = {
 
 void dcrypt_openssl_init(struct module *module ATTR_UNUSED)
 {
-	OpenSSL_add_all_algorithms();
-	ERR_load_crypto_strings();
+	dovecot_openssl_common_global_ref();
 	dcrypt_set_vfs(&dcrypt_openssl_vfs);
 }
 
 void dcrypt_openssl_deinit(void)
 {
-#if OPENSSL_API_COMPAT < 0x10100000L
-	OBJ_cleanup();
-#endif
+	dovecot_openssl_common_global_unref();
 }
diff --git a/src/lib-ssl-iostream/Makefile.am b/src/lib-ssl-iostream/Makefile.am
index 7b4ff2c3097..769de87b283 100644
--- a/src/lib-ssl-iostream/Makefile.am
+++ b/src/lib-ssl-iostream/Makefile.am
@@ -10,8 +10,13 @@ AM_CPPFLAGS = \
 if BUILD_OPENSSL
 module_LTLIBRARIES = libssl_iostream_openssl.la
 
+noinst_LTLIBRARIES += libdovecot_openssl_common.la
+libdovecot_openssl_common_la_LIBADD = $(SSL_LIBS)
+libdovecot_openssl_common_la_SOURCES = \
+	dovecot-openssl-common.c
+
 libssl_iostream_openssl_la_LDFLAGS = -module -avoid-version
-libssl_iostream_openssl_la_LIBADD = $(SSL_LIBS)
+libssl_iostream_openssl_la_LIBADD = libdovecot_openssl_common.la
 libssl_iostream_openssl_la_SOURCES = \
 	iostream-openssl.c \
 	iostream-openssl-common.c \
@@ -25,6 +30,9 @@ libssl_iostream_la_SOURCES = \
 	iostream-ssl.c \
 	$(ssl_sources)
 
+noinst_HEADERS = \
+	dovecot-openssl-common.h
+
 headers = \
 	iostream-openssl.h \
 	iostream-ssl.h \
diff --git a/src/lib-ssl-iostream/dovecot-openssl-common.c b/src/lib-ssl-iostream/dovecot-openssl-common.c
new file mode 100644
index 00000000000..7e622fb0459
--- /dev/null
+++ b/src/lib-ssl-iostream/dovecot-openssl-common.c
@@ -0,0 +1,81 @@
+/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "dovecot-openssl-common.h"
+
+#include <openssl/ssl.h>
+#include <openssl/engine.h>
+#include <openssl/rand.h>
+
+static int openssl_init_refcount = 0;
+static ENGINE *dovecot_openssl_engine;
+
+void dovecot_openssl_common_global_ref(void)
+{
+	unsigned char buf;
+
+	if (openssl_init_refcount++ > 0)
+		return;
+
+	SSL_library_init();
+	SSL_load_error_strings();
+	OpenSSL_add_all_algorithms();
+
+	/* PRNG initialization might want to use /dev/urandom, make sure it
+	   does it before chrooting. We might not have enough entropy at
+	   the first try, so this function may fail. It's still been
+	   initialized though. */
+	(void)RAND_bytes(&buf, 1);
+}
+
+bool dovecot_openssl_common_global_unref(void)
+{
+	i_assert(openssl_init_refcount > 0);
+
+	if (--openssl_init_refcount > 0)
+		return TRUE;
+
+	if (dovecot_openssl_engine != NULL) {
+		ENGINE_finish(dovecot_openssl_engine);
+		dovecot_openssl_engine = NULL;
+	}
+#if OPENSSL_VERSION_NUMBER < 0x10001000L
+	OBJ_cleanup();
+#endif
+#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+	SSL_COMP_free_compression_methods();
+#endif
+	ENGINE_cleanup();
+	EVP_cleanup();
+	CRYPTO_cleanup_all_ex_data();
+	ERR_remove_state(0);
+	ERR_free_strings();
+	return FALSE;
+}
+
+int dovecot_openssl_common_global_set_engine(const char *engine,
+					     const char **error_r)
+{
+	if (dovecot_openssl_engine != NULL)
+		return 1;
+
+	ENGINE_load_builtin_engines();
+	dovecot_openssl_engine = ENGINE_by_id(engine);
+	if (dovecot_openssl_engine == NULL) {
+		*error_r = t_strdup_printf("Unknown engine '%s'", engine);
+		return 0;
+	}
+	if (ENGINE_init(dovecot_openssl_engine) == 0) {
+		*error_r = t_strdup_printf("ENGINE_init(%s) failed", engine);
+		ENGINE_free(dovecot_openssl_engine);
+		dovecot_openssl_engine = NULL;
+		return -1;
+	}
+	if (ENGINE_set_default_RSA(dovecot_openssl_engine) == 0)
+		i_unreached();
+	if (ENGINE_set_default_DSA(dovecot_openssl_engine) == 0)
+		i_unreached();
+	if (ENGINE_set_default_ciphers(dovecot_openssl_engine) == 0)
+		i_unreached();
+	return 1;
+}
diff --git a/src/lib-ssl-iostream/dovecot-openssl-common.h b/src/lib-ssl-iostream/dovecot-openssl-common.h
new file mode 100644
index 00000000000..31854d3b1b6
--- /dev/null
+++ b/src/lib-ssl-iostream/dovecot-openssl-common.h
@@ -0,0 +1,16 @@
+#ifndef DOVECOT_OPENSSL_COMMON_H
+#define DOVECOT_OPENSSL_COMMON_H
+
+/* Initialize OpenSSL if this is the first instance.
+   Increase initialization reference count. */
+void dovecot_openssl_common_global_ref(void);
+/* Deinitialize OpenSSL if this is the last instance. Returns TRUE if there
+   are more instances left. */
+bool dovecot_openssl_common_global_unref(void);
+
+/* Set OpenSSL engine if it's not already set. Returns 1 on success, 0 if engine
+   is unknown, -1 on other error. error_r is set on 0/-1. */
+int dovecot_openssl_common_global_set_engine(const char *engine,
+					     const char **error_r);
+
+#endif
diff --git a/src/lib-ssl-iostream/iostream-openssl-context.c b/src/lib-ssl-iostream/iostream-openssl-context.c
index b976a0c03cd..2b6199df91d 100644
--- a/src/lib-ssl-iostream/iostream-openssl-context.c
+++ b/src/lib-ssl-iostream/iostream-openssl-context.c
@@ -3,14 +3,13 @@
 #include "lib.h"
 #include "safe-memset.h"
 #include "iostream-openssl.h"
+#include "dovecot-openssl-common.h"
 
 #include <openssl/crypto.h>
 #include <openssl/x509.h>
-#include <openssl/engine.h>
 #include <openssl/pem.h>
 #include <openssl/ssl.h>
 #include <openssl/err.h>
-#include <openssl/rand.h>
 
 #if !defined(OPENSSL_NO_ECDH) && OPENSSL_VERSION_NUMBER >= 0x10000000L
 #  define HAVE_ECDH
@@ -22,7 +21,6 @@ struct ssl_iostream_password_context {
 };
 
 static bool ssl_global_initialized = FALSE;
-static ENGINE *ssl_iostream_engine;
 int dovecot_ssl_extdata_index;
 
 static int ssl_iostream_init_global(const struct ssl_iostream_settings *set,
@@ -542,52 +540,36 @@ void openssl_iostream_context_deinit(struct ssl_iostream_context *ctx)
 
 void openssl_iostream_global_deinit(void)
 {
-	if (ssl_iostream_engine != NULL)
-		ENGINE_finish(ssl_iostream_engine);
-	ENGINE_cleanup();
-	EVP_cleanup();
-	CRYPTO_cleanup_all_ex_data();
-	ERR_remove_state(0);
-	ERR_free_strings();
+	dovecot_openssl_common_global_unref();
 }
 
 static int ssl_iostream_init_global(const struct ssl_iostream_settings *set,
 				    const char **error_r)
 {
 	static char dovecot[] = "dovecot";
-	unsigned char buf;
+	const char *error;
 
 	if (ssl_global_initialized)
 		return 0;
 
 	ssl_global_initialized = TRUE;
-	SSL_library_init();
-	SSL_load_error_strings();
-	OpenSSL_add_all_algorithms();
+	dovecot_openssl_common_global_ref();
 
 	dovecot_ssl_extdata_index =
 		SSL_get_ex_new_index(0, dovecot, NULL, NULL, NULL);
 
-	/* PRNG initialization might want to use /dev/urandom, make sure it
-	   does it before chrooting. We might not have enough entropy at
-	   the first try, so this function may fail. It's still been
-	   initialized though. */
-	(void)RAND_bytes(&buf, 1);
-
 	if (set->crypto_device != NULL && *set->crypto_device != '\0') {
-		ENGINE_load_builtin_engines();
-		ssl_iostream_engine = ENGINE_by_id(set->crypto_device);
-		if (ssl_iostream_engine == NULL) {
-			*error_r = t_strdup_printf(
+		switch (dovecot_openssl_common_global_set_engine(set->crypto_device, &error)) {
+		case 0:
+			error = t_strdup_printf(
 				"Unknown ssl_crypto_device: %s",
 				set->crypto_device);
+			/* fall through */
+		case -1:
+			*error_r = error;
 			/* we'll deinit at exit in any case */
 			return -1;
 		}
-		ENGINE_init(ssl_iostream_engine);
-		ENGINE_set_default_RSA(ssl_iostream_engine);
-		ENGINE_set_default_DSA(ssl_iostream_engine);
-		ENGINE_set_default_ciphers(ssl_iostream_engine);
 	}
 	return 0;
 }

From 44911be2c997639ca0dc2200138cfa04c7c4ac7b Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 13:33:05 +0300
Subject: [PATCH 347/450] lib-dcrypt: Added library dependencies to unit tests

---
 src/lib-dcrypt/Makefile.am | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/lib-dcrypt/Makefile.am b/src/lib-dcrypt/Makefile.am
index af25c10b53a..b51e774ab4a 100644
--- a/src/lib-dcrypt/Makefile.am
+++ b/src/lib-dcrypt/Makefile.am
@@ -48,15 +48,19 @@ check-test: all-am
 	  if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
 	done
 
-LIBDOVECOT_TEST = \
+LIBDOVECOT_TEST_DEPS = \
 	../lib-test/libtest.la \
-	../lib/liblib.la \
+	../lib/liblib.la
+LIBDOVECOT_TEST = \
+	$(LIBDOVECOT_TEST_DEPS) \
 	$(MODULE_LIBS)
 
 test_crypto_LDADD = $(LIBDOVECOT_TEST)
+test_crypto_DEPENDENCIES = $(LIBDOVECOT_TEST_DEPS)
 test_crypto_CFLAGS = $(AM_CPPFLAGS) -DDCRYPT_MODULE_DIR=\".libs\" -DDCRYPT_SRC_DIR=\"$(top_srcdir)/src/lib-dcrypt\"
 test_crypto_SOURCES = $(libdcrypt_la_SOURCES) test-crypto.c
 
 test_stream_LDADD = $(LIBDOVECOT_TEST)
+test_stream_DEPENDENCIES = $(LIBDOVECOT_TEST_DEPS)
 test_stream_CFLAGS = $(AM_CPPFLAGS) -DDCRYPT_MODULE_DIR=\".libs\" -DDCRYPT_SRC_DIR=\"$(top_srcdir)/src/lib-dcrypt\"
 test_stream_SOURCES = $(libdcrypt_la_SOURCES) test-stream.c

From f5e6b05684328b9800ccd973c73027300c832d65 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 13:33:34 +0300
Subject: [PATCH 348/450] lib-dcrypt: Allow specifying crypto_device (OpenSSL
 engine).

---
 src/lib-dcrypt/dcrypt-openssl.c | 11 +++++++++++
 src/lib-dcrypt/dcrypt-private.h |  2 ++
 src/lib-dcrypt/dcrypt.c         | 12 +++++++++++-
 src/lib-dcrypt/dcrypt.h         |  7 ++++++-
 src/lib-dcrypt/test-crypto.c    |  2 +-
 src/lib-dcrypt/test-stream.c    |  2 +-
 6 files changed, 32 insertions(+), 4 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index e7cfbb5374d..67baf713cf1 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -134,6 +134,16 @@ bool dcrypt_openssl_error(const char **error_r)
 	return FALSE;
 }
 
+static
+bool dcrypt_openssl_initialize(const struct dcrypt_settings *set, const char **error_r)
+{
+	if (set->crypto_device != NULL && set->crypto_device[0] != '\0') {
+		if (dovecot_openssl_common_global_set_engine(set->crypto_device, error_r) <= 0)
+			return FALSE;
+	}
+	return TRUE;
+}
+
 /* legacy function for old formats that generates
    hex encoded point from EC public key
  */
@@ -2006,6 +2016,7 @@ bool dcrypt_openssl_private_key_id(struct dcrypt_private_key *key, const char *a
 
 
 static struct dcrypt_vfs dcrypt_openssl_vfs = {
+	.initialize = dcrypt_openssl_initialize,
 	.ctx_sym_create = dcrypt_openssl_ctx_sym_create,
 	.ctx_sym_destroy = dcrypt_openssl_ctx_sym_destroy,
 	.ctx_sym_set_key = dcrypt_openssl_ctx_sym_set_key,
diff --git a/src/lib-dcrypt/dcrypt-private.h b/src/lib-dcrypt/dcrypt-private.h
index 523b03b0cbb..c8d1e121e9a 100644
--- a/src/lib-dcrypt/dcrypt-private.h
+++ b/src/lib-dcrypt/dcrypt-private.h
@@ -9,6 +9,8 @@
 #define DCRYPT_DOVECOT_KEY_ENCRYPT_PASSWORD 2
 
 struct dcrypt_vfs {
+	bool (*initialize)(const struct dcrypt_settings *set, const char **error_r);
+
 	bool (*ctx_sym_create)(const char *algorithm,
 		enum dcrypt_sym_mode mode,
 		struct dcrypt_context_symmetric **ctx_r, const char **error_r);
diff --git a/src/lib-dcrypt/dcrypt.c b/src/lib-dcrypt/dcrypt.c
index fa5b4275156..76468991fe8 100644
--- a/src/lib-dcrypt/dcrypt.c
+++ b/src/lib-dcrypt/dcrypt.c
@@ -5,8 +5,9 @@
 
 static struct module *dcrypt_module = NULL;
 static struct dcrypt_vfs *dcrypt_vfs = NULL;
+static const struct dcrypt_settings dcrypt_default_set;
 
-bool dcrypt_initialize(const char *backend, const char **error_r)
+bool dcrypt_initialize(const char *backend, const struct dcrypt_settings *set, const char **error_r)
 {
 	struct module_dir_load_settings mod_set;
 	const char *error;
@@ -15,6 +16,8 @@ bool dcrypt_initialize(const char *backend, const char **error_r)
 		return TRUE;
 	}
 	if (backend == NULL) backend = "openssl"; /* default for now */
+	if (set == NULL)
+		set = &dcrypt_default_set;
 
 	const char *implementation = t_strconcat("dcrypt_",backend,NULL);
 
@@ -29,6 +32,12 @@ bool dcrypt_initialize(const char *backend, const char **error_r)
 	}
 	module_dir_init(dcrypt_module);
 	i_assert(dcrypt_vfs != NULL);
+	if (dcrypt_vfs->initialize != NULL) {
+		if (!dcrypt_vfs->initialize(set, error_r)) {
+			dcrypt_deinitialize();
+			return FALSE;
+		}
+	}
 	/* Destroy SSL module after(most of) the others. Especially lib-fs
 	   backends may still want to access SSL module in their own
 	   atexit-callbacks. */
@@ -39,6 +48,7 @@ bool dcrypt_initialize(const char *backend, const char **error_r)
 void dcrypt_deinitialize(void)
 {
 	module_dir_unload(&dcrypt_module);
+	dcrypt_vfs = NULL;
 }
 
 void dcrypt_set_vfs(struct dcrypt_vfs *vfs)
diff --git a/src/lib-dcrypt/dcrypt.h b/src/lib-dcrypt/dcrypt.h
index 70c4d896607..94f8b4c2b2c 100644
--- a/src/lib-dcrypt/dcrypt.h
+++ b/src/lib-dcrypt/dcrypt.h
@@ -49,10 +49,15 @@ enum dcrypt_key_kind {
 	DCRYPT_KEY_KIND_PRIVATE
 };
 
+struct dcrypt_settings {
+	/* OpenSSL engine to use */
+	const char *crypto_device;
+};
+
 /**
  * load and initialize dcrypt backend, use either openssl or gnutls
  */
-bool dcrypt_initialize(const char *backend, const char **error_r);
+bool dcrypt_initialize(const char *backend, const struct dcrypt_settings *set, const char **error_r);
 
 /**
  * deinitialize dcrypt
diff --git a/src/lib-dcrypt/test-crypto.c b/src/lib-dcrypt/test-crypto.c
index a7199dedc4b..3af434b0885 100644
--- a/src/lib-dcrypt/test-crypto.c
+++ b/src/lib-dcrypt/test-crypto.c
@@ -308,7 +308,7 @@ void test_load_v2_public_key(void)
 }
 
 int main(void) {
-	dcrypt_initialize("openssl", NULL);
+	dcrypt_initialize("openssl", NULL, NULL);
 	random_init();
 	static void (*test_functions[])(void) = {
 		test_cipher_test_vectors,
diff --git a/src/lib-dcrypt/test-stream.c b/src/lib-dcrypt/test-stream.c
index 959f4cab034..2384ba29246 100644
--- a/src/lib-dcrypt/test-stream.c
+++ b/src/lib-dcrypt/test-stream.c
@@ -217,7 +217,7 @@ void test_write_read_v2(void)
 }
 
 int main(void) {
-	dcrypt_initialize("openssl", NULL);
+	dcrypt_initialize("openssl", NULL, NULL);
 	random_init();
 
 	dcrypt_key_load_private(&test_v1_kp.priv, DCRYPT_FORMAT_PEM, key_v1_priv, NULL, NULL, NULL);

From 344ac89667b41386e1ef46b6cc8e9c099fe4ada1 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 13:37:13 +0300
Subject: [PATCH 349/450] openssl: Use our own malloc()/realloc() that will die
 on out-of-memory.

This is likely safer rather than caller thinking for example that some input
is invalid when it's only a temporary memory allocation problem.
---
 src/lib-ssl-iostream/dovecot-openssl-common.c | 29 +++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/src/lib-ssl-iostream/dovecot-openssl-common.c b/src/lib-ssl-iostream/dovecot-openssl-common.c
index 7e622fb0459..b76c3ab6e56 100644
--- a/src/lib-ssl-iostream/dovecot-openssl-common.c
+++ b/src/lib-ssl-iostream/dovecot-openssl-common.c
@@ -10,6 +10,28 @@
 static int openssl_init_refcount = 0;
 static ENGINE *dovecot_openssl_engine;
 
+static void *dovecot_openssl_malloc(size_t size)
+{
+	/* this may be performance critical, so don't use
+	   i_malloc() or calloc() */
+	void *mem = malloc(size);
+	if (mem == NULL) {
+		i_fatal_status(FATAL_OUTOFMEM,
+			"OpenSSL: malloc(%"PRIuSIZE_T"): Out of memory", size);
+	}
+	return mem;
+}
+
+static void *dovecot_openssl_realloc(void *ptr, size_t size)
+{
+	void *mem = realloc(ptr, size);
+	if (mem == NULL) {
+		i_fatal_status(FATAL_OUTOFMEM,
+			"OpenSSL: realloc(%"PRIuSIZE_T"): Out of memory", size);
+	}
+	return mem;
+}
+
 void dovecot_openssl_common_global_ref(void)
 {
 	unsigned char buf;
@@ -17,6 +39,13 @@ void dovecot_openssl_common_global_ref(void)
 	if (openssl_init_refcount++ > 0)
 		return;
 
+	/* use our own memory allocation functions that will die instead of
+	   returning NULL. this avoids random failures on out-of-memory
+	   conditions. */
+	if (CRYPTO_set_mem_functions(dovecot_openssl_malloc,
+				     dovecot_openssl_realloc, free) == 0)
+		i_warning("CRYPTO_set_mem_functions() was called too late");
+
 	SSL_library_init();
 	SSL_load_error_strings();
 	OpenSSL_add_all_algorithms();

From 1ad12f870d2953c8e94795fd41bb1762fa89a5cd Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 14:29:57 +0300
Subject: [PATCH 350/450] Added valgrind suppression for OpenSSL <v1.0.2

This one works at least for CentOS 6. Because the OpenSSL library is
loaded as a plugin and unloaded before exit, valgrind sees only "???"
as the addresses. That's why we're using "obj:*". But the number of
obj:* lines is also important, so this may not work everywhere.
---
 run-test-valgrind.supp | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/run-test-valgrind.supp b/run-test-valgrind.supp
index 25aa3a2513f..64a2207a5da 100644
--- a/run-test-valgrind.supp
+++ b/run-test-valgrind.supp
@@ -15,3 +15,20 @@
    fun:lzma_stream_encoder
    fun:lzma_easy_encoder
 }
+{
+   <openssl_centos6_v1_0_1_compression_methods>
+   Memcheck:Leak
+   fun:malloc
+   obj:*
+   obj:*
+   obj:*
+   obj:*
+   obj:*
+   obj:*
+   obj:*
+   obj:*
+   obj:*
+   fun:module_dir_init
+   fun:dcrypt_initialize
+   fun:main
+}

From e28debf6ba906eb62a595d966671918396f5a888 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 17:52:28 +0300
Subject: [PATCH 351/450] lib: file_create_locked() - Unexpectedly deleted temp
 file is error.

This shouldn't happen and we shouldn't be silently retrying if it does.
---
 src/lib/file-create-locked.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/lib/file-create-locked.c b/src/lib/file-create-locked.c
index 3ea4c66b703..905e8cdc01f 100644
--- a/src/lib/file-create-locked.c
+++ b/src/lib/file-create-locked.c
@@ -69,9 +69,11 @@ try_create_new(const char *path, const struct file_create_settings *set,
 			/* just created by somebody else */
 			ret = 0;
 		} else if (errno == ENOENT) {
-			/* our temp file was just deleted by somebody else,
-			   retry creating it. */
-			ret = 0;
+			/* nobody should be deleting the temp file unless the
+			   entire directory is deleted. */
+			*error_r = t_strdup_printf(
+				"Temporary file %s was unexpectedly deleted",
+				str_c(temp_path));
 		} else {
 			*error_r = t_strdup_printf("link(%s, %s) failed: %m",
 						   str_c(temp_path), path);

From 80951e8dfe68116f4bc8f4b2adae6c8f1fe26fde Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 18:46:04 +0300
Subject: [PATCH 352/450] Updated run-test-valgrind.supp

---
 run-test-valgrind.supp | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/run-test-valgrind.supp b/run-test-valgrind.supp
index 64a2207a5da..0ccf1c67b4c 100644
--- a/run-test-valgrind.supp
+++ b/run-test-valgrind.supp
@@ -16,7 +16,7 @@
    fun:lzma_easy_encoder
 }
 {
-   <openssl_centos6_v1_0_1_compression_methods>
+   <openssl_centos6_i386_v1_0_1_compression_methods>
    Memcheck:Leak
    fun:malloc
    obj:*
@@ -32,3 +32,19 @@
    fun:dcrypt_initialize
    fun:main
 }
+{
+   <openssl_centos6_v1_0_1_compression_methods>
+   Memcheck:Leak
+   fun:malloc
+   obj:*
+   obj:*
+   obj:*
+   obj:*
+   obj:*
+   obj:*
+   obj:*
+   obj:*
+   fun:module_dir_init
+   fun:dcrypt_initialize
+   fun:main
+}

From 9a0dba9572713651f480cb81cf79061ebd198cbb Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 21:38:56 +0300
Subject: [PATCH 353/450] Makefile: Added run-test-valgrind.supp to EXTRA_DIST

---
 Makefile.am | 1 +
 1 file changed, 1 insertion(+)

diff --git a/Makefile.am b/Makefile.am
index a7be896d2fd..bc49ef8acde 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -14,6 +14,7 @@ EXTRA_DIST = \
 	ChangeLog \
 	cc-wrapper.sh.in \
 	update-version.sh \
+	run-test-valgrind.supp \
 	$(conf_DATA)
 
 noinst_DATA = dovecot-config

From 9412c4829857112df05e3be1c5421c0dd8770b02 Mon Sep 17 00:00:00 2001
From: Baofeng Wang <baofeng.wang@dovecot.fi>
Date: Tue, 7 Jun 2016 16:01:03 +0300
Subject: [PATCH 354/450] lib-test: add test_assert_failed_strcmp to expose
 strings.

the macro is used to facilitate any test to show string comparision
contents when they doesn't match.
---
 src/lib-test/test-common.c | 9 +++++++++
 src/lib-test/test-common.h | 9 +++++++++
 2 files changed, 18 insertions(+)

diff --git a/src/lib-test/test-common.c b/src/lib-test/test-common.c
index 04e61197acc..270e0effff4 100644
--- a/src/lib-test/test-common.c
+++ b/src/lib-test/test-common.c
@@ -179,6 +179,15 @@ void test_assert_failed_idx(const char *code, const char *file, unsigned int lin
 	test_success = FALSE;
 }
 
+void test_assert_failed_strcmp(const char *code, const char *file, unsigned int line,
+				const char * src, const char * dst)
+{
+	printf("%s: Assert(#%u) failed: %s\n", file, line, code);
+	printf("        \"%s\" != \"%s\"\n", src, dst);
+	fflush(stdout);
+	test_success = FALSE;
+}
+
 static void
 test_dump_rand_state(void)
 {
diff --git a/src/lib-test/test-common.h b/src/lib-test/test-common.h
index cbb02100738..fc62afb52da 100644
--- a/src/lib-test/test-common.h
+++ b/src/lib-test/test-common.h
@@ -17,8 +17,17 @@ void test_begin(const char *name);
 #define test_assert_idx(code, i) STMT_START { \
 		if (!(code)) test_assert_failed_idx(#code, __FILE__, __LINE__, i); \
 	} STMT_END
+/* Additional parameters are s1 (source) and s2 (destination) string
+ * in strcmp().
+ */
+#define test_assert_strcmp(s1, s2) STMT_START { \
+		if ((strcmp(s1,s2) != 0)) test_assert_failed_strcmp("strcmp(" #s1 ","  #s2 ")", __FILE__, __LINE__, s1, s2); \
+	} STMT_END
+
 void test_assert_failed(const char *code, const char *file, unsigned int line);
 void test_assert_failed_idx(const char *code, const char *file, unsigned int line, long long i);
+void test_assert_failed_strcmp(const char *code, const char *file, unsigned int line,
+				const char * src, const char * dst);
 bool test_has_failed(void);
 /* If you're testing nasty cases which you want to warn, surround the noisy op with these */
 void test_expect_errors(unsigned int expected);

From fb237dd841b49ac6a5cb4bf5228991518d882707 Mon Sep 17 00:00:00 2001
From: Baofeng Wang <baofeng.wang@dovecot.fi>
Date: Tue, 7 Jun 2016 15:58:38 +0300
Subject: [PATCH 355/450] lib-fts: remove trailing period character from
 email-address

any trailing period character '.' should be removed when email
tokenization is done.
---
 src/lib-fts/fts-tokenizer-address.c | 17 +++++++------
 src/lib-fts/fts-tokenizer-common.c  | 10 ++++++++
 src/lib-fts/fts-tokenizer-common.h  |  3 +++
 src/lib-fts/test-fts-tokenizer.c    | 39 +++++++++++++++++++++--------
 4 files changed, 51 insertions(+), 18 deletions(-)

diff --git a/src/lib-fts/fts-tokenizer-address.c b/src/lib-fts/fts-tokenizer-address.c
index 96f10eccddc..13c201271c6 100644
--- a/src/lib-fts/fts-tokenizer-address.c
+++ b/src/lib-fts/fts-tokenizer-address.c
@@ -79,6 +79,9 @@ static void
 fts_tokenizer_address_current_token(struct email_address_fts_tokenizer *tok,
                                     const char **token_r)
 {
+	const unsigned char *data = tok->last_word->data;
+	size_t len = tok->last_word->used;
+
 	tok->tokenizer.skip_parents = TRUE;
 	tok->state = EMAIL_ADDRESS_PARSER_STATE_NONE;
 	if (str_len(tok->last_word) > tok->max_length) {
@@ -86,15 +89,15 @@ fts_tokenizer_address_current_token(struct email_address_fts_tokenizer *tok,
 		/* As future proofing, delete partial utf8.
 		   IS_DTEXT() does not actually allow utf8 addresses
 		   yet though. */
-		const unsigned char *data = tok->last_word->data;
-		size_t len = tok->last_word->used;
+		len = tok->last_word->used;
 		fts_tokenizer_delete_trailing_partial_char(data, &len);
 		i_assert(len <= tok->max_length);
-		*token_r = len == 0 ? "" :
-			t_strndup(tok->last_word->data, len);
-	} else {
-		*token_r = t_strdup(str_c(tok->last_word));
 	}
+
+	if (len > 0)
+		fts_tokenizer_delete_trailing_invalid_char(data, &len);
+	*token_r = len == 0 ? "" :
+		t_strndup(data, len);
 }
 
 static bool
@@ -189,7 +192,7 @@ fts_tokenizer_email_address_parse_domain(struct email_address_fts_tokenizer *tok
 	while (pos < size && (IS_DTEXT(data[pos]) || data[pos] == '.'))
 		pos++;
 	 /* A complete domain name */
-	if ((pos > 1 && pos < size) || /* non-atext after atext in this data*/
+	if ((pos > 0 && pos < size) || /* non-atext after atext in this data*/
 	    (pos < size && !domain_is_empty(tok))) { /* non-atext after previous atext */
 		str_append_n(tok->last_word, data, pos);
 		*skip_r = pos;
diff --git a/src/lib-fts/fts-tokenizer-common.c b/src/lib-fts/fts-tokenizer-common.c
index f71113d036d..87faa7e9c3e 100644
--- a/src/lib-fts/fts-tokenizer-common.c
+++ b/src/lib-fts/fts-tokenizer-common.c
@@ -20,3 +20,13 @@ fts_tokenizer_delete_trailing_partial_char(const unsigned char *data,
 		*len = pos;
 	}
 }
+void fts_tokenizer_delete_trailing_invalid_char(const unsigned char *data,
+		   size_t *len)
+{
+	size_t pos = *len;
+
+	/* the token may contain '.' in the end - remove all of them. */
+	while (pos > 0 && data[pos-1] == '.')
+	    pos--;
+	*len = pos;
+}
diff --git a/src/lib-fts/fts-tokenizer-common.h b/src/lib-fts/fts-tokenizer-common.h
index fdd3b163137..b90e54353e9 100644
--- a/src/lib-fts/fts-tokenizer-common.h
+++ b/src/lib-fts/fts-tokenizer-common.h
@@ -3,4 +3,7 @@
 void
 fts_tokenizer_delete_trailing_partial_char(const unsigned char *data,
                                            size_t *len);
+void
+fts_tokenizer_delete_trailing_invalid_char(const unsigned char *data,
+		   size_t *len);
 #endif
diff --git a/src/lib-fts/test-fts-tokenizer.c b/src/lib-fts/test-fts-tokenizer.c
index 06e14ed8f83..960e5119d3c 100644
--- a/src/lib-fts/test-fts-tokenizer.c
+++ b/src/lib-fts/test-fts-tokenizer.c
@@ -4,16 +4,20 @@
 #include "unichar.h"
 #include "test-common.h"
 #include "fts-tokenizer.h"
+#include "fts-tokenizer-common.h"
 #include "fts-tokenizer-private.h"
 #include "fts-tokenizer-generic-private.h"
 
-
+/*there should be a trailing space ' ' at the end of each string except the last one*/
 #define TEST_INPUT_ADDRESS \
 	"@invalid invalid@ Abc Dfg <abc.dfg@example.com>, " \
 	"Bar Baz <bar@example.org>" \
 	"Foo Bar (comment)foo.bar@host.example.org " \
 	"foo, foo@domain " \
-	"abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.tld"
+	"abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.tld " \
+	"trailing, period@blue.com. " \
+	"multi-trialing, mul@trail.com..... " \
+	"m@s"
 
 static const char *test_inputs[] = {
 	/* generic things and word truncation: */
@@ -78,11 +82,11 @@ test_tokenizer_inputoutput(struct fts_tokenizer *tok, const char *_input,
 	/* test all input at once */
 	outi = first_outi;
 	while (fts_tokenizer_next(tok, input, input_len, &token, &error) > 0) {
-		test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi);
+		test_assert_strcmp(token, expected_output[outi]);
 		outi++;
 	}
 	while (fts_tokenizer_next(tok, NULL, 0, &token, &error) > 0) {
-		test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi);
+		test_assert_strcmp(token, expected_output[outi]);
 		outi++;
 	}
 	test_assert_idx(expected_output[outi] == NULL, outi);
@@ -92,12 +96,12 @@ test_tokenizer_inputoutput(struct fts_tokenizer *tok, const char *_input,
 	for (i = 0; i < input_len; i += char_len) {
 		char_len = uni_utf8_char_bytes(input[i]);
 		while (fts_tokenizer_next(tok, input+i, char_len, &token, &error) > 0) {
-			test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi);
+			test_assert_strcmp(token, expected_output[outi]);
 			outi++;
 		}
 	}
 	while (fts_tokenizer_final(tok, &token, &error) > 0) {
-		test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi);
+		test_assert_strcmp(token, expected_output[outi]);
 		outi++;
 	}
 	test_assert_idx(expected_output[outi] == NULL, outi);
@@ -109,12 +113,12 @@ test_tokenizer_inputoutput(struct fts_tokenizer *tok, const char *_input,
 		for (char_len = 0; char_len < max; )
 			char_len += uni_utf8_char_bytes(input[i+char_len]);
 		while (fts_tokenizer_next(tok, input+i, char_len, &token, &error) > 0) {
-			test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi);
+			test_assert_strcmp(token, expected_output[outi]);
 			outi++;
 		}
 	}
 	while (fts_tokenizer_final(tok, &token, &error) > 0) {
-		test_assert_idx(strcmp(token, expected_output[outi]) == 0, outi);
+		test_assert_strcmp(token, expected_output[outi]);
 		outi++;
 	}
 	test_assert_idx(expected_output[outi] == NULL, outi);
@@ -309,7 +313,11 @@ static void test_fts_tokenizer_address_only(void)
 	static const char *const expected_output[] = {
 		"abc.dfg@example.com", "bar@example.org",
 		"foo.bar@host.example.org", "foo@domain",
-		"abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstu", NULL
+		"abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstu",
+		"period@blue.com", /*trailing period '.' in email */
+		"mul@trail.com",
+		"m@s", /*one letter local-part and domain name */
+		NULL
 	};
 	struct fts_tokenizer *tok;
 	const char *error;
@@ -328,7 +336,11 @@ static void test_fts_tokenizer_address_parent(const char *name, const char * con
 		"invalid", "invalid", "Abc", "Dfg", "abc", "dfg", "example", "com", "abc.dfg@example.com",
 		"Bar", "Baz", "bar", "example", "org", "bar@example.org",
 		"Foo", "Bar", "comment", "foo", "bar", "host", "example", "org", "foo.bar@host.example.org",
-		"foo", "foo", "domain", "foo@domain", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyzabcde",  "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz","tld", "abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstu",  NULL
+		"foo", "foo", "domain", "foo@domain", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyzabcde",  "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz", "abcdefghijklmnopqrstuvxyz","tld", "abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstu",
+		"trailing", "period", "blue", "com", "period@blue.com",
+		"multi", "trialing", "mul", "trail", "com", "mul@trail.com",
+		"m", "s", "m@s",
+		NULL
 	};
 	struct fts_tokenizer *tok, *gen_tok;
 	const char *error;
@@ -360,7 +372,11 @@ static void test_fts_tokenizer_address_search(void)
 		"invalid", "invalid", "Abc", "Dfg", "abc.dfg@example.com",
 		"Bar", "Baz", "bar@example.org",
 		"Foo", "Bar", "comment", "foo.bar@host.example.org",
-		"foo", "foo@domain", "abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstu", NULL
+		"foo", "foo@domain", "abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstu",
+		"trailing", "period@blue.com",
+		"multi", "trialing", "mul@trail.com",
+		"m@s",
+		NULL
 	};
 	static const char *const settings[] = { "search", "", NULL };
 	struct fts_tokenizer *tok, *gen_tok;
@@ -418,5 +434,6 @@ int main(void)
 	fts_tokenizers_init();
 	ret = test_run(test_functions);
 	fts_tokenizers_deinit();
+
 	return ret;
 }

From 5406c10120b4ef8d4d138cdfa94f122434b4a9b2 Mon Sep 17 00:00:00 2001
From: Baofeng Wang <baofeng.wang@dovecot.fi>
Date: Tue, 7 Jun 2016 16:37:27 +0300
Subject: [PATCH 356/450] lib-fts: allow hyphen character in domain part

Allow hyphen character and remove possible trailing hyhpen character
when email tokenization is done.
---
 src/lib-fts/fts-tokenizer-address.c |  2 +-
 src/lib-fts/fts-tokenizer-common.c  |  3 ++-
 src/lib-fts/test-fts-tokenizer.c    | 10 +++++++++-
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/lib-fts/fts-tokenizer-address.c b/src/lib-fts/fts-tokenizer-address.c
index 13c201271c6..2b9380c499a 100644
--- a/src/lib-fts/fts-tokenizer-address.c
+++ b/src/lib-fts/fts-tokenizer-address.c
@@ -189,7 +189,7 @@ fts_tokenizer_email_address_parse_domain(struct email_address_fts_tokenizer *tok
 {
 	size_t pos = 0;
 
-	while (pos < size && (IS_DTEXT(data[pos]) || data[pos] == '.'))
+	while (pos < size && (IS_DTEXT(data[pos]) || data[pos] == '.' || data[pos] == '-'))
 		pos++;
 	 /* A complete domain name */
 	if ((pos > 0 && pos < size) || /* non-atext after atext in this data*/
diff --git a/src/lib-fts/fts-tokenizer-common.c b/src/lib-fts/fts-tokenizer-common.c
index 87faa7e9c3e..92feb71c28a 100644
--- a/src/lib-fts/fts-tokenizer-common.c
+++ b/src/lib-fts/fts-tokenizer-common.c
@@ -26,7 +26,8 @@ void fts_tokenizer_delete_trailing_invalid_char(const unsigned char *data,
 	size_t pos = *len;
 
 	/* the token may contain '.' in the end - remove all of them. */
-	while (pos > 0 && data[pos-1] == '.')
+	while (pos > 0 &&
+		  (data[pos-1] == '.' || data[pos-1] == '-'))
 	    pos--;
 	*len = pos;
 }
diff --git a/src/lib-fts/test-fts-tokenizer.c b/src/lib-fts/test-fts-tokenizer.c
index 960e5119d3c..6360a86f04c 100644
--- a/src/lib-fts/test-fts-tokenizer.c
+++ b/src/lib-fts/test-fts-tokenizer.c
@@ -17,7 +17,9 @@
 	"abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyzabcdefghijklmnopqrstuvxyz@abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.abcdefghijklmnopqrstuvxyz.tld " \
 	"trailing, period@blue.com. " \
 	"multi-trialing, mul@trail.com..... " \
-	"m@s"
+	"m@s " \
+	"hypen@hypen-hypen.com " \
+	"hypen@hypen-hypen-sick.com.-"
 
 static const char *test_inputs[] = {
 	/* generic things and word truncation: */
@@ -317,6 +319,8 @@ static void test_fts_tokenizer_address_only(void)
 		"period@blue.com", /*trailing period '.' in email */
 		"mul@trail.com",
 		"m@s", /*one letter local-part and domain name */
+		"hypen@hypen-hypen.com",
+		"hypen@hypen-hypen-sick.com",
 		NULL
 	};
 	struct fts_tokenizer *tok;
@@ -340,6 +344,8 @@ static void test_fts_tokenizer_address_parent(const char *name, const char * con
 		"trailing", "period", "blue", "com", "period@blue.com",
 		"multi", "trialing", "mul", "trail", "com", "mul@trail.com",
 		"m", "s", "m@s",
+		"hypen", "hypen", "hypen", "com", "hypen@hypen-hypen.com",
+		"hypen", "hypen", "hypen", "sick", "com", "hypen@hypen-hypen-sick.com",
 		NULL
 	};
 	struct fts_tokenizer *tok, *gen_tok;
@@ -376,6 +382,8 @@ static void test_fts_tokenizer_address_search(void)
 		"trailing", "period@blue.com",
 		"multi", "trialing", "mul@trail.com",
 		"m@s",
+		"hypen@hypen-hypen.com",
+		"hypen@hypen-hypen-sick.com",
 		NULL
 	};
 	static const char *const settings[] = { "search", "", NULL };

From 1fb947e5ea792aafb06e07cec86afce4b78de8bc Mon Sep 17 00:00:00 2001
From: Baofeng Wang <baofeng.wang@dovecot.fi>
Date: Wed, 8 Jun 2016 16:41:04 +0300
Subject: [PATCH 357/450] lib-fts: add unit test to
 fts_tokenizer_delete_trailing_partial_char()

---
 src/lib-fts/test-fts-tokenizer.c | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/lib-fts/test-fts-tokenizer.c b/src/lib-fts/test-fts-tokenizer.c
index 6360a86f04c..06fa79041c7 100644
--- a/src/lib-fts/test-fts-tokenizer.c
+++ b/src/lib-fts/test-fts-tokenizer.c
@@ -424,6 +424,18 @@ static void test_fts_tokenizer_address_search(void)
 	test_end();
 }
 
+static void test_fts_tokenizer_delete_trailing_partial_char()
+{
+	const char* str[] = {"\x7f", "\xC2\x80", "\xE0\x80\x80","\xF0\x80\x80\x80"};
+	unsigned int i;
+	size_t new_size;
+	for (i = 0; i < 4; i++) {
+		new_size = i+1;
+		fts_tokenizer_delete_trailing_partial_char((unsigned char*)str[i], &new_size);
+		test_assert( i+1 == new_size);
+	}
+}
+
 int main(void)
 {
 	static void (*test_functions[])(void) = {
@@ -435,6 +447,7 @@ int main(void)
 		test_fts_tokenizer_address_parent_simple,
 		test_fts_tokenizer_address_parent_tr29,
 		test_fts_tokenizer_address_search,
+		test_fts_tokenizer_delete_trailing_partial_char,
 		NULL
 	};
 	int ret;

From 8425e55f5aa8926ea00aadf7273c42b1bab8c781 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 21 Jun 2016 12:04:24 +0300
Subject: [PATCH 358/450] lib-dict: Explicitly specify used dict_vfuncs methods
 for drivers.

This allows adding more methods without modifying all the existing drivers.
---
 src/lib-dict/dict-cdb.c             | 17 +++-------------
 src/lib-dict/dict-client.c          | 31 ++++++++++++++---------------
 src/lib-dict/dict-db.c              | 25 +++++++++++------------
 src/lib-dict/dict-file.c            | 29 ++++++++++++---------------
 src/lib-dict/dict-memcached-ascii.c | 26 ++++++++++--------------
 src/lib-dict/dict-memcached.c       | 18 +++--------------
 src/lib-dict/dict-redis.c           | 26 ++++++++++--------------
 src/lib-dict/dict-sql.c             | 31 ++++++++++++++---------------
 8 files changed, 81 insertions(+), 122 deletions(-)

diff --git a/src/lib-dict/dict-cdb.c b/src/lib-dict/dict-cdb.c
index 755dbde5622..25e1def4e03 100644
--- a/src/lib-dict/dict-cdb.c
+++ b/src/lib-dict/dict-cdb.c
@@ -122,20 +122,9 @@ static int cdb_dict_lookup(struct dict *_dict, pool_t pool,
 struct dict dict_driver_cdb = {
 	.name = "cdb",
 	{
-		cdb_dict_init,
-		cdb_dict_deinit,
-		NULL,
-		cdb_dict_lookup,
-		NULL,
-		NULL,
-		NULL,
-		NULL,
-		NULL,
-		NULL,
-		NULL,
-		NULL,
-		NULL,
-		NULL
+		.init = cdb_dict_init,
+		.deinit = cdb_dict_deinit,
+		.lookup = cdb_dict_lookup,
 	}
 };
 #endif
diff --git a/src/lib-dict/dict-client.c b/src/lib-dict/dict-client.c
index 032ae0c3491..916504a927f 100644
--- a/src/lib-dict/dict-client.c
+++ b/src/lib-dict/dict-client.c
@@ -1007,21 +1007,20 @@ struct dict dict_driver_client = {
 	.name = "proxy",
 
 	{
-		client_dict_init,
-		client_dict_deinit,
-		client_dict_wait,
-		client_dict_lookup,
-		client_dict_iterate_init,
-		client_dict_iterate,
-		client_dict_iterate_deinit,
-		client_dict_transaction_init,
-		client_dict_transaction_commit,
-		client_dict_transaction_rollback,
-		client_dict_set,
-		client_dict_unset,
-		NULL,
-		client_dict_atomic_inc,
-		client_dict_lookup_async,
-		client_dict_switch_ioloop
+		.init = client_dict_init,
+		.deinit = client_dict_deinit,
+		.wait = client_dict_wait,
+		.lookup = client_dict_lookup,
+		.iterate_init = client_dict_iterate_init,
+		.iterate = client_dict_iterate,
+		.iterate_deinit = client_dict_iterate_deinit,
+		.transaction_init = client_dict_transaction_init,
+		.transaction_commit = client_dict_transaction_commit,
+		.transaction_rollback = client_dict_transaction_rollback,
+		.set = client_dict_set,
+		.unset = client_dict_unset,
+		.atomic_inc = client_dict_atomic_inc,
+		.lookup_async = client_dict_lookup_async,
+		.switch_ioloop = client_dict_switch_ioloop
 	}
 };
diff --git a/src/lib-dict/dict-db.c b/src/lib-dict/dict-db.c
index 729824b5f49..541665b2b31 100644
--- a/src/lib-dict/dict-db.c
+++ b/src/lib-dict/dict-db.c
@@ -465,19 +465,18 @@ db_dict_atomic_inc(struct dict_transaction_context *_ctx ATTR_UNUSED,
 struct dict dict_driver_db = {
 	.name = "db",
 	{
-		db_dict_init,
-		db_dict_deinit,
-		NULL,
-		db_dict_lookup,
-		db_dict_iterate_init,
-		db_dict_iterate,
-		db_dict_iterate_deinit,
-		db_dict_transaction_init,
-		db_dict_transaction_commit,
-		db_dict_transaction_rollback,
-		db_dict_set,
-		db_dict_unset,
-		db_dict_atomic_inc
+		.init = db_dict_init,
+		.deinit = db_dict_deinit,
+		.lookup = db_dict_lookup,
+		.iterate_init = db_dict_iterate_init,
+		.iterate = db_dict_iterate,
+		.iterate_deinit = db_dict_iterate_deinit,
+		.transaction_init = db_dict_transaction_init,
+		.transaction_commit = db_dict_transaction_commit,
+		.transaction_rollback = db_dict_transaction_rollback,
+		.set = db_dict_set,
+		.unset = db_dict_unset,
+		.atomic_inc = db_dict_atomic_inc,
 	}
 };
 #endif
diff --git a/src/lib-dict/dict-file.c b/src/lib-dict/dict-file.c
index e22465ee20d..ae9aa4aa01f 100644
--- a/src/lib-dict/dict-file.c
+++ b/src/lib-dict/dict-file.c
@@ -649,21 +649,18 @@ file_dict_transaction_commit(struct dict_transaction_context *_ctx,
 struct dict dict_driver_file = {
 	.name = "file",
 	{
-		file_dict_init,
-		file_dict_deinit,
-		NULL,
-		file_dict_lookup,
-		file_dict_iterate_init,
-		file_dict_iterate,
-		file_dict_iterate_deinit,
-		file_dict_transaction_init,
-		file_dict_transaction_commit,
-		dict_transaction_memory_rollback,
-		dict_transaction_memory_set,
-		dict_transaction_memory_unset,
-		dict_transaction_memory_append,
-		dict_transaction_memory_atomic_inc,
-		NULL,
-		NULL
+		.init = file_dict_init,
+		.deinit = file_dict_deinit,
+		.lookup = file_dict_lookup,
+		.iterate_init = file_dict_iterate_init,
+		.iterate = file_dict_iterate,
+		.iterate_deinit = file_dict_iterate_deinit,
+		.transaction_init = file_dict_transaction_init,
+		.transaction_commit = file_dict_transaction_commit,
+		.transaction_rollback = dict_transaction_memory_rollback,
+		.set = dict_transaction_memory_set,
+		.unset = dict_transaction_memory_unset,
+		.append = dict_transaction_memory_append,
+		.atomic_inc = dict_transaction_memory_atomic_inc,
 	}
 };
diff --git a/src/lib-dict/dict-memcached-ascii.c b/src/lib-dict/dict-memcached-ascii.c
index ebb27871ba3..efcabd9f1af 100644
--- a/src/lib-dict/dict-memcached-ascii.c
+++ b/src/lib-dict/dict-memcached-ascii.c
@@ -650,21 +650,15 @@ memcached_ascii_transaction_commit(struct dict_transaction_context *_ctx,
 struct dict dict_driver_memcached_ascii = {
 	.name = "memcached_ascii",
 	{
-		memcached_ascii_dict_init,
-		memcached_ascii_dict_deinit,
-		NULL,
-		memcached_ascii_dict_lookup,
-		NULL,
-		NULL,
-		NULL,
-		memcached_ascii_transaction_init,
-		memcached_ascii_transaction_commit,
-		dict_transaction_memory_rollback,
-		dict_transaction_memory_set,
-		dict_transaction_memory_unset,
-		dict_transaction_memory_append,
-		dict_transaction_memory_atomic_inc,
-		NULL,
-		NULL
+		.init = memcached_ascii_dict_init,
+		.deinit = memcached_ascii_dict_deinit,
+		.lookup = memcached_ascii_dict_lookup,
+		.transaction_init = memcached_ascii_transaction_init,
+		.transaction_commit = memcached_ascii_transaction_commit,
+		.transaction_rollback = dict_transaction_memory_rollback,
+		.set = dict_transaction_memory_set,
+		.unset = dict_transaction_memory_unset,
+		.append = dict_transaction_memory_append,
+		.atomic_inc = dict_transaction_memory_atomic_inc,
 	}
 };
diff --git a/src/lib-dict/dict-memcached.c b/src/lib-dict/dict-memcached.c
index e3bd6067fe4..eafd22584fe 100644
--- a/src/lib-dict/dict-memcached.c
+++ b/src/lib-dict/dict-memcached.c
@@ -377,20 +377,8 @@ static int memcached_dict_lookup(struct dict *_dict, pool_t pool,
 struct dict dict_driver_memcached = {
 	.name = "memcached",
 	{
-		memcached_dict_init,
-		memcached_dict_deinit,
-		NULL,
-		memcached_dict_lookup,
-		NULL,
-		NULL,
-		NULL,
-		NULL,
-		NULL,
-		NULL,
-		NULL,
-		NULL,
-		NULL,
-		NULL,
-		NULL
+		.init = memcached_dict_init,
+		.deinit = memcached_dict_deinit,
+		.lookup = memcached_dict_lookup,
 	}
 };
diff --git a/src/lib-dict/dict-redis.c b/src/lib-dict/dict-redis.c
index b7e92ce9285..77e3a1dd389 100644
--- a/src/lib-dict/dict-redis.c
+++ b/src/lib-dict/dict-redis.c
@@ -790,21 +790,15 @@ static void redis_atomic_inc(struct dict_transaction_context *_ctx,
 struct dict dict_driver_redis = {
 	.name = "redis",
 	{
-		redis_dict_init,
-		redis_dict_deinit,
-		NULL,
-		redis_dict_lookup,
-		NULL,
-		NULL,
-		NULL,
-		redis_transaction_init,
-		redis_transaction_commit,
-		redis_transaction_rollback,
-		redis_set,
-		redis_unset,
-		redis_append,
-		redis_atomic_inc,
-		NULL,
-		NULL
+		.init = redis_dict_init,
+		.deinit = redis_dict_deinit,
+		.lookup = redis_dict_lookup,
+		.transaction_init = redis_transaction_init,
+		.transaction_commit = redis_transaction_commit,
+		.transaction_rollback = redis_transaction_rollback,
+		.set = redis_set,
+		.unset = redis_unset,
+		.append = redis_append,
+		.atomic_inc = redis_atomic_inc,
 	}
 };
diff --git a/src/lib-dict/dict-sql.c b/src/lib-dict/dict-sql.c
index b142e360e18..feb46a67c9a 100644
--- a/src/lib-dict/dict-sql.c
+++ b/src/lib-dict/dict-sql.c
@@ -1246,22 +1246,21 @@ static struct dict sql_dict = {
 	.name = "sql",
 
 	{
-		sql_dict_init,
-		sql_dict_deinit,
-		sql_dict_wait,
-		sql_dict_lookup,
-		sql_dict_iterate_init,
-		sql_dict_iterate,
-		sql_dict_iterate_deinit,
-		sql_dict_transaction_init,
-		sql_dict_transaction_commit,
-		sql_dict_transaction_rollback,
-		sql_dict_set,
-		sql_dict_unset,
-		sql_dict_append,
-		sql_dict_atomic_inc,
-		sql_dict_lookup_async,
-		NULL
+		.init = sql_dict_init,
+		.deinit = sql_dict_deinit,
+		.wait = sql_dict_wait,
+		.lookup = sql_dict_lookup,
+		.iterate_init = sql_dict_iterate_init,
+		.iterate = sql_dict_iterate,
+		.iterate_deinit = sql_dict_iterate_deinit,
+		.transaction_init = sql_dict_transaction_init,
+		.transaction_commit = sql_dict_transaction_commit,
+		.transaction_rollback = sql_dict_transaction_rollback,
+		.set = sql_dict_set,
+		.unset = sql_dict_unset,
+		.append = sql_dict_append,
+		.atomic_inc = sql_dict_atomic_inc,
+		.lookup_async = sql_dict_lookup_async,
 	}
 };
 

From ce3cb757be6f8959236db19abab4615c21bb8602 Mon Sep 17 00:00:00 2001
From: Baofeng Wang <baofeng.wang@dovecot.fi>
Date: Mon, 20 Jun 2016 15:10:55 +0300
Subject: [PATCH 359/450] lib-storage: Add autoexpunge_max_mails configuration
 to autoexpunge

Mails are expunged until mail count is at autoexpunge_max_mails or below.

In below example, autoexpunge will expunge 1 mail when message count > 100
and *then* try to expunge mails that are still older than 2 minutes:

namespace {
  ..
   mailbox Trash {
     autoexpunge = 2 mins
     autoexpunge_max_mails = 100
   }
}
---
 src/lib-storage/mail-autoexpunge.c      | 51 ++++++++++++++++++-------
 src/lib-storage/mail-storage-settings.c |  4 +-
 src/lib-storage/mail-storage-settings.h |  1 +
 3 files changed, 41 insertions(+), 15 deletions(-)

diff --git a/src/lib-storage/mail-autoexpunge.c b/src/lib-storage/mail-autoexpunge.c
index 840c184ba4a..a9d0054043d 100644
--- a/src/lib-storage/mail-autoexpunge.c
+++ b/src/lib-storage/mail-autoexpunge.c
@@ -8,20 +8,35 @@
 #include "mail-user.h"
 #include "mail-autoexpunge.h"
 
-static int mailbox_autoexpunge(struct mailbox *box, time_t expire_time)
+static int
+mailbox_autoexpunge(struct mailbox *box, unsigned int interval_time,
+		    unsigned int max_mails)
 {
 	struct mailbox_transaction_context *t;
 	struct mail *mail;
 	struct mailbox_metadata metadata;
 	const struct mail_index_header *hdr;
+	struct mailbox_status status;
 	uint32_t seq;
-	time_t timestamp;
+	time_t timestamp, expire_time;
 	int ret = 0;
 
+	if ((unsigned int)ioloop_time < interval_time)
+		expire_time = 0;
+	else
+		expire_time = ioloop_time - interval_time;
+
 	/* first try to check quickly from mailbox list index if we should
 	   bother opening this mailbox. */
-	if (mailbox_get_metadata(box, MAILBOX_METADATA_FIRST_SAVE_DATE,
-				 &metadata) == 0) {
+	if (mailbox_get_status(box, STATUS_MESSAGES, &status) < 0)
+		return -1;
+	if (interval_time == 0 && status.messages <= max_mails)
+		return 0;
+
+	if (max_mails == 0 || status.messages <= max_mails) {
+		if (mailbox_get_metadata(box, MAILBOX_METADATA_FIRST_SAVE_DATE,
+					 &metadata) < 0)
+			return -1;
 		if (metadata.first_save_date == (time_t)-1 ||
 		    metadata.first_save_date > expire_time)
 			return 0;
@@ -41,7 +56,15 @@ static int mailbox_autoexpunge(struct mailbox *box, time_t expire_time)
 	hdr = mail_index_get_header(box->view);
 	for (seq = 1; seq <= hdr->messages_count; seq++) {
 		mail_set_seq(mail, seq);
-		if (mail_get_save_date(mail, &timestamp) == 0) {
+		if (max_mails > 0 && hdr->messages_count - seq + 1 > max_mails) {
+			/* max_mails is still being reached -> expunge.
+			   don't even check saved-dates before we're
+			   below max_mails. */
+			mail_expunge(mail);
+		} else if (interval_time == 0) {
+			/* only max_mails is used. nothing further to do. */
+			break;
+		} else if (mail_get_save_date(mail, &timestamp) == 0) {
 			if (timestamp > expire_time)
 				break;
 			mail_expunge(mail);
@@ -61,18 +84,16 @@ static int mailbox_autoexpunge(struct mailbox *box, time_t expire_time)
 
 static void
 mailbox_autoexpunge_set(struct mail_namespace *ns, const char *vname,
-			unsigned int autoexpunge)
+			unsigned int autoexpunge,
+			unsigned int autoexpunge_max_mails)
 {
 	struct mailbox *box;
-	time_t expire_time;
-
-	expire_time = ioloop_time - autoexpunge;
 
 	/* autoexpunge is configured by admin, so we can safely ignore
 	   any ACLs the user might normally have against expunging in
 	   the mailbox. */
 	box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_IGNORE_ACLS);
-	if (mailbox_autoexpunge(box, expire_time) < 0) {
+	if (mailbox_autoexpunge(box, autoexpunge, autoexpunge_max_mails) < 0) {
 		i_error("Failed to autoexpunge mailbox '%s': %s",
 			mailbox_get_vname(box),
 			mailbox_get_last_error(box, NULL));
@@ -92,7 +113,8 @@ mailbox_autoexpunge_wildcards(struct mail_namespace *ns,
 				      MAILBOX_LIST_ITER_SKIP_ALIASES |
 				      MAILBOX_LIST_ITER_RETURN_NO_FLAGS);
 	while ((info = mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
-		mailbox_autoexpunge_set(ns, info->vname, set->autoexpunge);
+		mailbox_autoexpunge_set(ns, info->vname, set->autoexpunge,
+					set->autoexpunge_max_mails);
 	} T_END;
 	if (mailbox_list_iter_deinit(&iter) < 0) {
 		i_error("Failed to iterate autoexpunge mailboxes '%s%s': %s",
@@ -110,8 +132,8 @@ static void mail_namespace_autoexpunge(struct mail_namespace *ns)
 		return;
 
 	array_foreach(&ns->set->mailboxes, box_set) {
-		if ((*box_set)->autoexpunge == 0 ||
-		    (unsigned int)ioloop_time < (*box_set)->autoexpunge)
+		if ((*box_set)->autoexpunge == 0 &&
+		    (*box_set)->autoexpunge_max_mails == 0)
 			continue;
 
 		if (strpbrk((*box_set)->name, "*?") != NULL)
@@ -122,7 +144,8 @@ static void mail_namespace_autoexpunge(struct mail_namespace *ns)
 				vname = t_strndup(ns->prefix, ns->prefix_len - 1);
 			else
 				vname = t_strconcat(ns->prefix, (*box_set)->name, NULL);
-			mailbox_autoexpunge_set(ns, vname, (*box_set)->autoexpunge);
+			mailbox_autoexpunge_set(ns, vname, (*box_set)->autoexpunge,
+						(*box_set)->autoexpunge_max_mails);
 		}
 	}
 }
diff --git a/src/lib-storage/mail-storage-settings.c b/src/lib-storage/mail-storage-settings.c
index e2233bfb7c5..d1998fa0566 100644
--- a/src/lib-storage/mail-storage-settings.c
+++ b/src/lib-storage/mail-storage-settings.c
@@ -127,6 +127,7 @@ static const struct setting_define mailbox_setting_defines[] = {
 	DEF(SET_STR, driver),
 	DEF(SET_STR, comment),
 	DEF(SET_TIME, autoexpunge),
+	DEF(SET_UINT, autoexpunge_max_mails),
 
 	SETTING_DEFINE_LIST_END
 };
@@ -139,7 +140,8 @@ const struct mailbox_settings mailbox_default_settings = {
 	.special_use = "",
 	.driver = "",
 	.comment = "",
-	.autoexpunge = 0
+	.autoexpunge = 0,
+	.autoexpunge_max_mails = 0
 };
 
 const struct setting_parser_info mailbox_setting_parser_info = {
diff --git a/src/lib-storage/mail-storage-settings.h b/src/lib-storage/mail-storage-settings.h
index 44196a2147d..152872fc6b0 100644
--- a/src/lib-storage/mail-storage-settings.h
+++ b/src/lib-storage/mail-storage-settings.h
@@ -82,6 +82,7 @@ struct mailbox_settings {
 	const char *driver;
 	const char *comment;
 	unsigned int autoexpunge;
+	unsigned int autoexpunge_max_mails;
 };
 
 struct mail_user_settings {

From d5fbefd8eae8de71afbf55673da3c3b9e04fba8f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 21 Jun 2016 16:49:19 +0300
Subject: [PATCH 360/450] fs-posix: Autodelete directories also when "prefix"
 parameter is used.

---
 src/lib-fs/fs-posix.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/lib-fs/fs-posix.c b/src/lib-fs/fs-posix.c
index de9f43ead75..3090f0d3405 100644
--- a/src/lib-fs/fs-posix.c
+++ b/src/lib-fs/fs-posix.c
@@ -201,12 +201,13 @@ static int fs_posix_rmdir_parents(struct posix_fs *fs, const char *path)
 {
 	const char *p;
 
-	if (fs->root_path == NULL)
+	if (fs->root_path == NULL && fs->path_prefix == NULL)
 		return 0;
 
 	while ((p = strrchr(path, '/')) != NULL) {
 		path = t_strdup_until(path, p);
-		if (strcmp(path, fs->root_path) == 0)
+		if ((fs->root_path != NULL && strcmp(path, fs->root_path) == 0) ||
+		    (fs->path_prefix != NULL && strncmp(path, fs->path_prefix, strlen(path)) == 0))
 			break;
 		if (rmdir(path) == 0) {
 			/* success, continue to parent */

From c39000f9aa595e4fc15825f7045fb073d79ece2d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 21 Jun 2016 17:24:11 +0300
Subject: [PATCH 361/450] fs-posix: Added "dirs" parameter to enable explicit
 directory removal.

---
 src/lib-fs/fs-posix.c | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/src/lib-fs/fs-posix.c b/src/lib-fs/fs-posix.c
index 3090f0d3405..646997504a1 100644
--- a/src/lib-fs/fs-posix.c
+++ b/src/lib-fs/fs-posix.c
@@ -35,6 +35,7 @@ struct posix_fs {
 	enum fs_posix_lock_method lock_method;
 	mode_t mode;
 	bool mode_auto;
+	bool have_dirs;
 };
 
 struct posix_fs_file {
@@ -104,6 +105,8 @@ fs_posix_init(struct fs *_fs, const char *args, const struct fs_settings *set)
 				fs->path_prefix = i_strdup(arg + 7);
 		} else if (strcmp(arg, "mode=auto") == 0) {
 			fs->mode_auto = TRUE;
+		} else if (strcmp(arg, "dirs") == 0) {
+			fs->have_dirs = TRUE;
 		} else if (strncmp(arg, "mode=", 5) == 0) {
 			unsigned int mode;
 			if (str_to_uint_oct(arg+5, &mode) < 0) {
@@ -133,13 +136,21 @@ static void fs_posix_deinit(struct fs *_fs)
 	i_free(fs);
 }
 
-static enum fs_properties fs_posix_get_properties(struct fs *fs ATTR_UNUSED)
+static enum fs_properties fs_posix_get_properties(struct fs *_fs)
 {
-	/* FS_PROPERTY_DIRECTORIES not returned because fs_delete()
-	   automatically rmdir()s parents. This could be changed later though,
-	   but SIS code at least would need to be changed to support it. */
-	return FS_PROPERTY_LOCKS | FS_PROPERTY_FASTCOPY | FS_PROPERTY_RENAME |
+	struct posix_fs *fs = (struct posix_fs *)_fs;
+	enum fs_properties props =
+		FS_PROPERTY_LOCKS | FS_PROPERTY_FASTCOPY | FS_PROPERTY_RENAME |
 		FS_PROPERTY_STAT | FS_PROPERTY_ITER | FS_PROPERTY_RELIABLEITER;
+
+	/* FS_PROPERTY_DIRECTORIES is not returned normally because fs_delete()
+	   automatically rmdir()s parents. For backwards compatibility
+	   (especially with SIS code) we'll do it that way, but optionally with
+	   "dirs" parameter enable them. This is especially important to be
+	   able to use doveadm fs commands to delete empty directories. */
+	if (fs->have_dirs)
+		props |= FS_PROPERTY_DIRECTORIES;
+	return props;
 }
 
 static int
@@ -201,6 +212,8 @@ static int fs_posix_rmdir_parents(struct posix_fs *fs, const char *path)
 {
 	const char *p;
 
+	if (fs->have_dirs)
+		return 0;
 	if (fs->root_path == NULL && fs->path_prefix == NULL)
 		return 0;
 

From 07a1f7b108d40e2cc68f2967cacd161b8a24108c Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 21 Jun 2016 17:26:01 +0300
Subject: [PATCH 362/450] fs-posix: Allow ":" as well as space as parameter
 separator.

This makes it consistent with all the other fs drivers.
---
 src/lib-fs/fs-posix.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-fs/fs-posix.c b/src/lib-fs/fs-posix.c
index 646997504a1..3acad9347fb 100644
--- a/src/lib-fs/fs-posix.c
+++ b/src/lib-fs/fs-posix.c
@@ -88,7 +88,7 @@ fs_posix_init(struct fs *_fs, const char *args, const struct fs_settings *set)
 	fs->lock_method = FS_POSIX_LOCK_METHOD_FLOCK;
 	fs->mode = FS_DEFAULT_MODE;
 
-	tmp = t_strsplit_spaces(args, " ");
+	tmp = t_strsplit_spaces(args, ": ");
 	for (; *tmp != NULL; tmp++) {
 		const char *arg = *tmp;
 

From 4559dcffe55351103026b71a3eba5a43146a6948 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martti=20Rannanj=C3=A4rvi?= <martti.rannanjarvi@dovecot.fi>
Date: Tue, 21 Jun 2016 15:11:04 +0300
Subject: [PATCH 363/450] doc: fix wrong tense in doveadm-expunge man page

---
 doc/man/doveadm-expunge.1.in | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/man/doveadm-expunge.1.in b/doc/man/doveadm-expunge.1.in
index 14b0f4318e6..41d7571775f 100644
--- a/doc/man/doveadm-expunge.1.in
+++ b/doc/man/doveadm-expunge.1.in
@@ -39,7 +39,7 @@ commands.
 .PP
 In the first form,
 .BR doveadm (1)
-will executed the
+will execute the
 .B expunge
 action with the environment of the logged in system user.
 .PP

From 525ddf5f27c256c3a8caaff2db3a6f835fd5fb2a Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 11:39:55 +0300
Subject: [PATCH 364/450] Added welcome plugin.

---
 configure.ac                         |   1 +
 src/plugins/Makefile.am              |   1 +
 src/plugins/welcome/Makefile.am      |  14 +++
 src/plugins/welcome/welcome-plugin.c | 146 +++++++++++++++++++++++++++
 4 files changed, 162 insertions(+)
 create mode 100644 src/plugins/welcome/Makefile.am
 create mode 100644 src/plugins/welcome/welcome-plugin.c

diff --git a/configure.ac b/configure.ac
index 060ab3acb36..39b0c86abe5 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2954,6 +2954,7 @@ src/plugins/stats/Makefile
 src/plugins/imap-stats/Makefile
 src/plugins/trash/Makefile
 src/plugins/virtual/Makefile
+src/plugins/welcome/Makefile
 src/plugins/zlib/Makefile
 src/plugins/imap-zlib/Makefile
 stamp.h
diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am
index 7e0d00b17d9..7dc31d954c6 100644
--- a/src/plugins/Makefile.am
+++ b/src/plugins/Makefile.am
@@ -39,6 +39,7 @@ SUBDIRS = \
 	imap-stats \
 	trash \
 	virtual \
+	welcome \
 	$(ZLIB) \
 	$(FTS_LUCENE) \
 	$(FTS_SOLR) \
diff --git a/src/plugins/welcome/Makefile.am b/src/plugins/welcome/Makefile.am
new file mode 100644
index 00000000000..98968f5dc0b
--- /dev/null
+++ b/src/plugins/welcome/Makefile.am
@@ -0,0 +1,14 @@
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage
+
+NOPLUGIN_LDFLAGS =
+lib99_welcome_plugin_la_LDFLAGS = -module -avoid-version
+
+module_LTLIBRARIES = \
+	lib99_welcome_plugin.la
+
+lib99_welcome_plugin_la_SOURCES = \
+	welcome-plugin.c
diff --git a/src/plugins/welcome/welcome-plugin.c b/src/plugins/welcome/welcome-plugin.c
new file mode 100644
index 00000000000..715529351fa
--- /dev/null
+++ b/src/plugins/welcome/welcome-plugin.c
@@ -0,0 +1,146 @@
+/* Copyright (c) 2015-2016 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "eacces-error.h"
+#include "write-full.h"
+#include "module-context.h"
+#include "mail-storage-private.h"
+
+#define WELCOME_SOCKET_TIMEOUT_SECS 30
+
+#define WELCOME_CONTEXT(obj) \
+	MODULE_CONTEXT(obj, welcome_storage_module)
+
+struct welcome_mailbox {
+	union mailbox_module_context module_ctx;
+	bool created;
+};
+
+static MODULE_CONTEXT_DEFINE_INIT(welcome_storage_module,
+				  &mail_storage_module_register);
+
+static void script_execute(struct mail_user *user, const char *cmd, bool wait)
+{
+	const char *socket_path, *const *args;
+	string_t *str;
+	char buf[1024];
+	int fd, ret;
+
+	if (user->mail_debug)
+		i_debug("welcome: Executing %s (wait=%d)", cmd, wait);
+
+	args = t_strsplit_spaces(cmd, " ");
+	socket_path = args[0];
+	args++;
+
+	if (*socket_path != '/') {
+		socket_path = t_strconcat(user->set->base_dir, "/",
+					  socket_path, NULL);
+	}
+	if ((fd = net_connect_unix_with_retries(socket_path, 1000)) < 0) {
+		if (errno == EACCES) {
+			i_error("welcome: %s",
+				eacces_error_get("net_connect_unix",
+						 socket_path));
+		} else {
+			i_error("welcome: net_connect_unix(%s) failed: %m",
+				socket_path);
+		}
+		return;
+	}
+
+	str = t_str_new(1024);
+	str_append(str, "VERSION\tscript\t3\t0\n");
+	if (!wait)
+		str_append(str, "noreply\n");
+	else
+		str_append(str, "-\n");
+	for (; *args != NULL; args++) {
+		str_append(str, *args);
+		str_append_c(str, '\n');
+	}
+	str_append_c(str, '\n');
+
+	alarm(WELCOME_SOCKET_TIMEOUT_SECS);
+	net_set_nonblock(fd, FALSE);
+	if (write_full(fd, str_data(str), str_len(str)) < 0)
+		i_error("write(%s) failed: %m", socket_path);
+	else if (wait) {
+		ret = read(fd, buf, sizeof(buf));
+		if (ret < 0)
+			i_error("welcome: read(%s) failed: %m", socket_path);
+		else if (ret < 2)
+			i_error("welcome: %s failed: Only %d bytes read", socket_path, ret);
+		else if (buf[0] != '+')
+			i_error("welcome: %s failed: Script returned error", socket_path);
+	}
+	if (close(fd) < 0)
+		i_error("close(%s) failed: %m", socket_path);
+}
+
+static int
+welcome_create_box(struct mailbox *box,
+		   const struct mailbox_update *update, bool directory)
+{
+	struct welcome_mailbox *wbox = WELCOME_CONTEXT(box);
+
+	if (wbox->module_ctx.super.create_box(box, update, directory) < 0)
+		return -1;
+	/* the mailbox isn't fully created here yet, so just mark it as created
+	   and wait until open() time to actually run it */
+	wbox->created = TRUE;
+	return 0;
+}
+
+static int welcome_open_box(struct mailbox *box)
+{
+	struct welcome_mailbox *wbox = WELCOME_CONTEXT(box);
+	const char *cmd;
+
+	cmd = !wbox->created ? NULL :
+		mail_user_plugin_getenv(box->storage->user, "welcome_script");
+	if (cmd != NULL) {
+		bool wait = mail_user_plugin_getenv(box->storage->user,
+						    "welcome_wait") != NULL;
+		script_execute(box->storage->user, cmd, wait);
+	}
+	return wbox->module_ctx.super.open(box);
+}
+
+static void welcome_mailbox_allocated(struct mailbox *box)
+{
+	struct mailbox_vfuncs *v = box->vlast;
+	struct welcome_mailbox *wbox;
+
+	if (!box->inbox_user)
+		return;
+
+	wbox = p_new(box->pool, struct welcome_mailbox, 1);
+	wbox->module_ctx.super = *v;
+	box->vlast = &wbox->module_ctx.super;
+
+	v->create_box = welcome_create_box;
+	v->open = welcome_open_box;
+	MODULE_CONTEXT_SET(box, welcome_storage_module, wbox);
+}
+
+static struct mail_storage_hooks welcome_mail_storage_hooks = {
+	.mailbox_allocated = welcome_mailbox_allocated
+};
+
+void welcome_plugin_init(struct module *module);
+void welcome_plugin_deinit(void);
+
+void welcome_plugin_init(struct module *module)
+{
+	mail_storage_hooks_add(module, &welcome_mail_storage_hooks);
+}
+
+void welcome_plugin_deinit(void)
+{
+	mail_storage_hooks_remove(&welcome_mail_storage_hooks);
+}
+
+const char *welcome_plugin_version = DOVECOT_ABI_VERSION;

From 15730f82f7efc564b570026a06ed05f6ac1495f4 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 20:06:38 +0300
Subject: [PATCH 365/450] dbox: Optimize POP3 MAIL_FETCH_UIDL_BACKEND.

We keep track of the highest UID known to have POP3 UIDL in index's header.
If saving adds a newer message, it'll also update the header. When fetching
UIDL_BACKEND, we can need to check only mails with lower UIDs. There are
some race conditions here, but normally UIDLs are set only once during
migration so it shouldn't matter.
---
 src/lib-storage/index/Makefile.am             |   2 +
 src/lib-storage/index/dbox-common/dbox-mail.c |  21 +++-
 src/lib-storage/index/dbox-common/dbox-save.c |   4 +
 src/lib-storage/index/dbox-common/dbox-save.h |   1 +
 src/lib-storage/index/dbox-multi/mdbox-save.c |  12 +++
 .../index/dbox-single/sdbox-save.c            |   5 +
 src/lib-storage/index/index-pop3-uidl.c       | 101 ++++++++++++++++++
 src/lib-storage/index/index-pop3-uidl.h       |  16 +++
 src/lib-storage/index/index-storage.c         |   3 +
 src/lib-storage/index/index-transaction.c     |   2 +
 src/lib-storage/mail-storage-private.h        |   8 ++
 11 files changed, 173 insertions(+), 2 deletions(-)
 create mode 100644 src/lib-storage/index/index-pop3-uidl.c
 create mode 100644 src/lib-storage/index/index-pop3-uidl.h

diff --git a/src/lib-storage/index/Makefile.am b/src/lib-storage/index/Makefile.am
index 6e2fa6f9cd7..220ccd7958a 100644
--- a/src/lib-storage/index/Makefile.am
+++ b/src/lib-storage/index/Makefile.am
@@ -20,6 +20,7 @@ libstorage_index_la_SOURCES = \
 	index-mail-binary.c \
 	index-mail-headers.c \
 	index-mailbox-size.c \
+	index-pop3-uidl.c \
 	index-rebuild.c \
 	index-search.c \
 	index-search-result.c \
@@ -41,6 +42,7 @@ headers = \
 	index-attachment.h \
 	index-mail.h \
 	index-mailbox-size.h \
+	index-pop3-uidl.h \
 	index-rebuild.h \
 	index-search-private.h \
 	index-search-result.h \
diff --git a/src/lib-storage/index/dbox-common/dbox-mail.c b/src/lib-storage/index/dbox-common/dbox-mail.c
index ddd93bdaa61..6785904be13 100644
--- a/src/lib-storage/index/dbox-common/dbox-mail.c
+++ b/src/lib-storage/index/dbox-common/dbox-mail.c
@@ -5,6 +5,7 @@
 #include "str.h"
 #include "index-storage.h"
 #include "index-mail.h"
+#include "index-pop3-uidl.h"
 #include "dbox-attachment.h"
 #include "dbox-storage.h"
 #include "dbox-file.h"
@@ -223,15 +224,31 @@ int dbox_mail_get_special(struct mail *_mail, enum mail_fetch_field field,
 			  const char **value_r)
 {
 	struct dbox_mail *mail = (struct dbox_mail *)_mail;
+	int ret;
 
 	/* keep the UIDL in cache file, otherwise POP3 would open all
 	   mail files and read the metadata. same for GUIDs if they're
 	   used. */
 	switch (field) {
 	case MAIL_FETCH_UIDL_BACKEND:
-		return dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_UIDL,
-						MAIL_CACHE_POP3_UIDL, value_r);
+		if (!index_pop3_uidl_can_exist(_mail)) {
+			*value_r = "";
+			return 0;
+		}
+		ret = dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_UIDL,
+					       MAIL_CACHE_POP3_UIDL, value_r);
+		if (ret == 0) {
+			index_pop3_uidl_update_exists(&mail->imail.mail.mail,
+						      (*value_r)[0] != '\0');
+		}
+		return ret;
 	case MAIL_FETCH_POP3_ORDER:
+		if (!index_pop3_uidl_can_exist(_mail)) {
+			/* we're assuming that if there's a POP3 order, there's
+			   also a UIDL */
+			*value_r = "";
+			return 0;
+		}
 		return dbox_get_cached_metadata(mail, DBOX_METADATA_POP3_ORDER,
 						MAIL_CACHE_POP3_ORDER, value_r);
 	case MAIL_FETCH_GUID:
diff --git a/src/lib-storage/index/dbox-common/dbox-save.c b/src/lib-storage/index/dbox-common/dbox-save.c
index fe305a6df89..456cfe339c9 100644
--- a/src/lib-storage/index/dbox-common/dbox-save.c
+++ b/src/lib-storage/index/dbox-common/dbox-save.c
@@ -166,11 +166,15 @@ void dbox_save_write_metadata(struct mail_save_context *_ctx,
 		str_printfa(str, "%c%s\n", DBOX_METADATA_POP3_UIDL,
 			    mdata->pop3_uidl);
 		ctx->have_pop3_uidls = TRUE;
+		ctx->highest_pop3_uidl_seq =
+			I_MAX(ctx->highest_pop3_uidl_seq, ctx->seq);
 	}
 	if (mdata->pop3_order != 0) {
 		str_printfa(str, "%c%u\n", DBOX_METADATA_POP3_ORDER,
 			    mdata->pop3_order);
 		ctx->have_pop3_orders = TRUE;
+		ctx->highest_pop3_uidl_seq =
+			I_MAX(ctx->highest_pop3_uidl_seq, ctx->seq);
 	}
 
 	guid = mdata->guid;
diff --git a/src/lib-storage/index/dbox-common/dbox-save.h b/src/lib-storage/index/dbox-common/dbox-save.h
index 7b16d1c87ae..9d5f17333c9 100644
--- a/src/lib-storage/index/dbox-common/dbox-save.h
+++ b/src/lib-storage/index/dbox-common/dbox-save.h
@@ -14,6 +14,7 @@ struct dbox_save_context {
 
 	struct ostream *dbox_output;
 
+	uint32_t highest_pop3_uidl_seq;
 	unsigned int failed:1;
 	unsigned int finished:1;
 	unsigned int have_pop3_uidls:1;
diff --git a/src/lib-storage/index/dbox-multi/mdbox-save.c b/src/lib-storage/index/dbox-multi/mdbox-save.c
index 857ff7b7b2f..d9bdc022b6a 100644
--- a/src/lib-storage/index/dbox-multi/mdbox-save.c
+++ b/src/lib-storage/index/dbox-multi/mdbox-save.c
@@ -11,6 +11,7 @@
 #include "ostream.h"
 #include "write-full.h"
 #include "index-mail.h"
+#include "index-pop3-uidl.h"
 #include "mail-copy.h"
 #include "dbox-save.h"
 #include "mdbox-storage.h"
@@ -326,6 +327,17 @@ int mdbox_transaction_save_commit_pre(struct mail_save_context *_ctx)
 	mail_index_append_finish_uids(ctx->ctx.trans, hdr->next_uid,
 				      &_t->changes->saved_uids);
 
+	if (ctx->ctx.highest_pop3_uidl_seq != 0) {
+		struct seq_range_iter iter;
+		uint32_t uid;
+
+		seq_range_array_iter_init(&iter, &_t->changes->saved_uids);
+		if (!seq_range_array_iter_nth(&iter,
+				ctx->ctx.highest_pop3_uidl_seq-1, &uid))
+			i_unreached();
+		index_pop3_uidl_set_max_uid(&ctx->mbox->box, ctx->ctx.trans, uid);
+	}
+
 	/* save map UIDs to mailbox index */
 	if (first_map_uid != 0)
 		mdbox_save_set_map_uids(ctx, first_map_uid, last_map_uid);
diff --git a/src/lib-storage/index/dbox-single/sdbox-save.c b/src/lib-storage/index/dbox-single/sdbox-save.c
index 9f0eff8852b..5560d0d7a21 100644
--- a/src/lib-storage/index/dbox-single/sdbox-save.c
+++ b/src/lib-storage/index/dbox-single/sdbox-save.c
@@ -12,6 +12,7 @@
 #include "write-full.h"
 #include "index-mail.h"
 #include "mail-copy.h"
+#include "index-pop3-uidl.h"
 #include "dbox-attachment.h"
 #include "dbox-save.h"
 #include "sdbox-storage.h"
@@ -248,6 +249,10 @@ static int dbox_save_assign_uids(struct sdbox_save_context *ctx,
 		i_assert(ret);
 		if (sdbox_file_assign_uid(sfile, uid) < 0)
 			return -1;
+		if (ctx->ctx.highest_pop3_uidl_seq == i+1) {
+			index_pop3_uidl_set_max_uid(&ctx->mbox->box,
+				ctx->ctx.trans, uid);
+		}
 	}
 	i_assert(!seq_range_array_iter_nth(&iter, n, &uid));
 	return 0;
diff --git a/src/lib-storage/index/index-pop3-uidl.c b/src/lib-storage/index/index-pop3-uidl.c
new file mode 100644
index 00000000000..5ebc7073ff6
--- /dev/null
+++ b/src/lib-storage/index/index-pop3-uidl.c
@@ -0,0 +1,101 @@
+/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "index-storage.h"
+#include "index-mail.h"
+#include "index-pop3-uidl.h"
+
+void index_pop3_uidl_set_max_uid(struct mailbox *box,
+				 struct mail_index_transaction *trans,
+				 uint32_t uid)
+{
+	struct mailbox_index_pop3_uidl uidl;
+
+	memset(&uidl, 0, sizeof(uidl));
+	uidl.max_uid_with_pop3_uidl = uid;
+
+	mail_index_update_header_ext(trans, box->pop3_uidl_hdr_ext_id,
+				     0, &uidl, sizeof(uidl));
+}
+
+bool index_pop3_uidl_can_exist(struct mail *mail)
+{
+	struct mailbox_index_pop3_uidl uidl;
+	const void *data;
+	size_t size;
+
+	/* We'll assume that if the header exists, it's up-to-date. normally
+	   UIDLs are set only during migration, so this value never changes.
+	   Also even if it does, it becomes out-of-date only when the mailbox
+	   is modified with old Dovecot versions. To fix that we'd have to
+	   add and keep updating "max tracked uid" in this header for every
+	   saved mail, which isn't worth it. */
+	mail_index_get_header_ext(mail->transaction->view,
+				  mail->box->pop3_uidl_hdr_ext_id,
+				  &data, &size);
+	if (size < sizeof(uidl)) {
+		/* this header isn't set yet */
+		return TRUE;
+	}
+	memcpy(&uidl, data, size);
+	return mail->uid <= uidl.max_uid_with_pop3_uidl;
+}
+
+void index_pop3_uidl_update_exists(struct mail *mail, bool exists)
+{
+	struct mailbox_transaction_context *trans = mail->transaction;
+
+	if (exists) {
+		if (trans->highest_pop3_uidl_uid < mail->uid) {
+			trans->highest_pop3_uidl_uid = mail->uid;
+			trans->prev_pop3_uidl_tracking_seq = mail->seq;
+		}
+	} else if (mail->seq == trans->prev_pop3_uidl_tracking_seq+1) {
+		trans->prev_pop3_uidl_tracking_seq++;
+	} else {
+		/* skipping mails. we don't know the state. */
+	}
+}
+
+void index_pop3_uidl_update_exists_finish(struct mailbox_transaction_context *trans)
+{
+	struct mail_index_view *view;
+	struct mailbox_index_pop3_uidl uidl;
+	const void *data;
+	size_t size;
+	bool seen_all_msgs;
+
+	if (trans->highest_pop3_uidl_uid == 0)
+		return;
+
+	/* First check that we actually looked at UIDL for all messages.
+	   Otherwise we can't say for sure if the newest messages had UIDLs. */
+	if (trans->prev_pop3_uidl_tracking_seq !=
+	    mail_index_view_get_messages_count(trans->view))
+		return;
+
+	/* Just to be sure: Refresh the index and check again. POP3 keeps
+	   transactions open for duration of the entire session. Maybe another
+	   process already added new mails (and already updated this header).
+	   This check is racy, but normally UIDLs aren't added after migration
+	   so it's a bit questionable if it's even worth having this check in
+	   there. */
+	view = mail_index_view_open(trans->box->index);
+	seen_all_msgs = mail_index_refresh(trans->box->index) == 0 &&
+		trans->prev_pop3_uidl_tracking_seq ==
+		mail_index_view_get_messages_count(view);
+	mail_index_view_close(&view);
+	if (!seen_all_msgs)
+		return;
+
+	/* check if we have already the same header */
+	mail_index_get_header_ext(trans->view, trans->box->pop3_uidl_hdr_ext_id,
+				  &data, &size);
+	if (size >= sizeof(uidl)) {
+		memcpy(&uidl, data, size);
+		if (trans->highest_pop3_uidl_uid == uidl.max_uid_with_pop3_uidl)
+			return;
+	}
+	index_pop3_uidl_set_max_uid(trans->box, trans->itrans,
+				    trans->highest_pop3_uidl_uid);
+}
diff --git a/src/lib-storage/index/index-pop3-uidl.h b/src/lib-storage/index/index-pop3-uidl.h
new file mode 100644
index 00000000000..956ab7ddd5f
--- /dev/null
+++ b/src/lib-storage/index/index-pop3-uidl.h
@@ -0,0 +1,16 @@
+#ifndef INDEX_POP3_H
+#define INDEX_POP3_H
+
+struct mail_index_transaction;
+struct mail;
+struct mailbox;
+struct mailbox_transaction_context;
+
+void index_pop3_uidl_set_max_uid(struct mailbox *box,
+				 struct mail_index_transaction *trans,
+				 uint32_t uid);
+bool index_pop3_uidl_can_exist(struct mail *mail);
+void index_pop3_uidl_update_exists(struct mail *mail, bool exists);
+void index_pop3_uidl_update_exists_finish(struct mailbox_transaction_context *trans);
+
+#endif
diff --git a/src/lib-storage/index/index-storage.c b/src/lib-storage/index/index-storage.c
index 339ebbb7d10..0d5a407ad5c 100644
--- a/src/lib-storage/index/index-storage.c
+++ b/src/lib-storage/index/index-storage.c
@@ -293,6 +293,9 @@ int index_storage_mailbox_open(struct mailbox *box, bool move_to_memory)
 		mail_index_ext_register(box->index, "hdr-vsize",
 					sizeof(struct mailbox_index_vsize), 0,
 					sizeof(uint64_t));
+	box->pop3_uidl_hdr_ext_id =
+		mail_index_ext_register(box->index, "hdr-pop3-uidl",
+					sizeof(struct mailbox_index_pop3_uidl), 0, 0);
 
 	box->opened = TRUE;
 
diff --git a/src/lib-storage/index/index-transaction.c b/src/lib-storage/index/index-transaction.c
index 4c7b42cc3b6..db1f21914e6 100644
--- a/src/lib-storage/index/index-transaction.c
+++ b/src/lib-storage/index/index-transaction.c
@@ -5,6 +5,7 @@
 #include "dict.h"
 #include "index-storage.h"
 #include "index-sync-private.h"
+#include "index-pop3-uidl.h"
 #include "index-mail.h"
 
 static void index_transaction_free(struct mailbox_transaction_context *t)
@@ -28,6 +29,7 @@ index_transaction_index_commit(struct mail_index_transaction *index_trans,
 	struct index_mailbox_sync_pvt_context *pvt_sync_ctx = NULL;
 	int ret = 0;
 
+	index_pop3_uidl_update_exists_finish(t);
 	if (t->nontransactional_changes)
 		t->changes->changed = TRUE;
 
diff --git a/src/lib-storage/mail-storage-private.h b/src/lib-storage/mail-storage-private.h
index 0aad8f83f80..4d8bba30972 100644
--- a/src/lib-storage/mail-storage-private.h
+++ b/src/lib-storage/mail-storage-private.h
@@ -292,6 +292,10 @@ struct mailbox_index_vsize {
 	uint32_t message_count;
 };
 
+struct mailbox_index_pop3_uidl {
+	uint32_t max_uid_with_pop3_uidl;
+};
+
 struct mailbox_index_first_saved {
 	uint32_t uid;
 	uint32_t timestamp;
@@ -346,6 +350,7 @@ struct mailbox {
 	enum mailbox_feature enabled_features;
 	struct mail_msgpart_partial_cache partial_cache;
 	uint32_t vsize_hdr_ext_id;
+	uint32_t pop3_uidl_hdr_ext_id;
 
 	/* MAIL_RECENT flags handling */
 	ARRAY_TYPE(seq_range) recent_flags;
@@ -551,6 +556,9 @@ struct mailbox_transaction_context {
 	struct mail_transaction_commit_changes *changes;
 	ARRAY(union mailbox_transaction_module_context *) module_contexts;
 
+	uint32_t prev_pop3_uidl_tracking_seq;
+	uint32_t highest_pop3_uidl_uid;
+
 	struct mail_save_context *save_ctx;
 	/* number of mails saved/copied within this transaction. */
 	unsigned int save_count;

From dc9e4ab2880482c69eb61951788092815b89cb2b Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 21 Jun 2016 18:30:20 +0300
Subject: [PATCH 366/450] doveadm: Fixed --long-parameters handling

---
 src/doveadm/doveadm-cmd.c | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/doveadm/doveadm-cmd.c b/src/doveadm/doveadm-cmd.c
index 8d9ff5092ee..0bb8dd408b9 100644
--- a/src/doveadm/doveadm-cmd.c
+++ b/src/doveadm/doveadm-cmd.c
@@ -378,18 +378,17 @@ doveadm_build_options(const struct doveadm_cmd_param par[],
 {
 	for(size_t i=0; par[i].name != NULL; i++) {
 		struct option longopt;
+
+		memset(&longopt, 0, sizeof(longopt));
 		longopt.name = par[i].name;
-		longopt.flag = 0;
-		longopt.val = 0;
 		if (par[i].short_opt != '\0') {
 			longopt.val = par[i].short_opt;
 			str_append_c(shortopts, par[i].short_opt);
 			if (par[i].type != CMD_PARAM_BOOL)
 				str_append_c(shortopts, ':');
-		} else {
-			if (par[i].type != CMD_PARAM_BOOL) longopt.has_arg = 1;
-			else longopt.has_arg = 0;
 		}
+		if (par[i].type != CMD_PARAM_BOOL)
+			longopt.has_arg = 1;
 		array_append(longopts, &longopt, 1);
 	}
 	array_append_zero(longopts);

From cb21fec783fa3a36d6dad08724634298d5caefee Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 21 Jun 2016 18:47:42 +0300
Subject: [PATCH 367/450] doveadm fs delete: Allow multiple paths also with -R
 parameter.

---
 src/doveadm/doveadm-fs.c | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/src/doveadm/doveadm-fs.c b/src/doveadm/doveadm-fs.c
index fec6da90771..e785844d992 100644
--- a/src/doveadm/doveadm-fs.c
+++ b/src/doveadm/doveadm-fs.c
@@ -398,16 +398,12 @@ cmd_fs_delete_dir_recursive(struct fs *fs, unsigned int async_count,
 	doveadm_fs_delete_async_finish(&ctx);
 }
 
-static void
-cmd_fs_delete_recursive(int argc, char *argv[], unsigned int async_count)
+static void cmd_fs_delete_recursive_path(struct fs *fs, const char *path,
+					 unsigned int async_count)
 {
-	struct fs *fs;
 	struct fs_file *file;
-	const char *path;
 	unsigned int path_len;
 
-	fs = cmd_fs_init(&argc, &argv, 1, cmd_fs_delete);
-	path = argv[0];
 	path_len = strlen(path);
 	if (path_len > 0 && path[path_len-1] != '/')
 		path = t_strconcat(path, "/", NULL);
@@ -423,6 +419,17 @@ cmd_fs_delete_recursive(int argc, char *argv[], unsigned int async_count)
 		}
 		fs_file_deinit(&file);
 	}
+}
+
+static void
+cmd_fs_delete_recursive(int argc, char *argv[], unsigned int async_count)
+{
+	struct fs *fs;
+	unsigned int i;
+
+	fs = cmd_fs_init(&argc, &argv, 0, cmd_fs_delete);
+	for (i = 0; argv[i] != NULL; i++)
+		cmd_fs_delete_recursive_path(fs, argv[i], async_count);
 	fs_deinit(&fs);
 }
 

From 83c5005bd00b2be5cf98ccec41e4a50083b5e51a Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 19 Jun 2016 20:48:27 +0300
Subject: [PATCH 368/450] lib-dcrypt: Assert-crash if impossible private/public
 keys are seen.

---
 src/lib-dcrypt/dcrypt-openssl.c | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index 67baf713cf1..99803f02aa2 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -1323,9 +1323,8 @@ bool dcrypt_openssl_encrypt_private_key_dovecot(buffer_t *key, int enctype, cons
 				return FALSE;
 			}
 		} else {
-			if (error_r != NULL)
-				*error_r = "Unsupported encryption key";
-			return FALSE;
+			/* Loading the key should have failed */
+			i_unreached();
 		}
 		/* add encryption key id, reuse peer_key buffer */
 	} else if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PASSWORD) {
@@ -1401,9 +1400,8 @@ bool dcrypt_openssl_store_private_key_dovecot(struct dcrypt_private_key *key, co
 		ptr = buffer_append_space_unsafe(buf, len);
 		BN_bn2mpi(pk, ptr);
 	} else {
-		if (*error_r != NULL)
-			*error_r = "Unsupported key type";
-		return FALSE;
+		/* Loading the key should have failed */
+		i_unreached();
 	}
 
 	/* see if we want ECDH based or password based encryption */
@@ -1640,9 +1638,8 @@ bool dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, s
 		EVP_PKEY_set1_EC_KEY(pk, eck);
 		EC_KEY_free(eck);
 	} else {
-		if (error_r != NULL)
-			*error_r = "Invalid private key";
-		return FALSE;
+		/* Loading the key should have failed */
+		i_unreached();
 	}
 
 	*pub_key_r = (struct dcrypt_public_key*)pk;
@@ -1869,6 +1866,7 @@ bool dcrypt_openssl_private_key_type(struct dcrypt_private_key *key, enum dcrypt
 	if (priv == NULL) return FALSE;
 	if (EVP_PKEY_base_id(priv) == EVP_PKEY_RSA) *key_type = DCRYPT_KEY_RSA;
 	else if (EVP_PKEY_base_id(priv) == EVP_PKEY_EC) *key_type = DCRYPT_KEY_EC;
+	else i_unreached();
 	return FALSE;
 }
 
@@ -1879,6 +1877,7 @@ bool dcrypt_openssl_public_key_type(struct dcrypt_public_key *key, enum dcrypt_k
 	if (pub == NULL) return FALSE;
 	if (EVP_PKEY_base_id(pub) == EVP_PKEY_RSA) *key_type = DCRYPT_KEY_RSA;
 	else if (EVP_PKEY_base_id(pub) == EVP_PKEY_EC) *key_type = DCRYPT_KEY_EC;
+	else i_unreached();
 	return FALSE;
 }
 

From 49046dc1102df9292c4d3da973b033bfa8e6cb49 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 19 Jun 2016 20:55:19 +0300
Subject: [PATCH 369/450] lib-dcrypt: Assert-crash if key parameter is NULL.

If it happens, it's a bug.
---
 src/lib-dcrypt/dcrypt-openssl.c | 30 +++++-------------------------
 1 file changed, 5 insertions(+), 25 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index 99803f02aa2..4e055b4ecda 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -1658,11 +1658,7 @@ bool dcrypt_openssl_key_string_get_info(const char *key_data, enum dcrypt_key_fo
 	char *encryption_key_hash = NULL;
 	char *key_hash = NULL;
 
-	if (key_data == NULL) {
-		if (error_r != NULL)
-			*error_r = "NULL key passed";
-		return FALSE;
-	}
+	i_assert(key_data != NULL);
 
 	/* is it PEM key */
 	if (strncmp(key_data, "-----BEGIN ", 11) == 0) {
@@ -1888,11 +1884,7 @@ bool dcrypt_openssl_public_key_id_old(struct dcrypt_public_key *key, buffer_t *r
 	unsigned char buf[SHA256_DIGEST_LENGTH];
 	EVP_PKEY *pub = (EVP_PKEY*)key;
 
-	if (pub == NULL) {
-		if (error_r != NULL)
-			*error_r = "key is NULL";
-		return FALSE;
-	}
+	i_assert(pub != NULL);
 	if (EVP_PKEY_base_id(pub) != EVP_PKEY_EC) {
 		if (error_r != NULL)
 			*error_r = "Only EC key supported";
@@ -1913,11 +1905,7 @@ bool dcrypt_openssl_private_key_id_old(struct dcrypt_private_key *key, buffer_t
 	unsigned char buf[SHA256_DIGEST_LENGTH];
 	EVP_PKEY *priv = (EVP_PKEY*)key;
 
-	if (priv == NULL) {
-		if (error_r != NULL)
-			*error_r = "key is NULL";
-		return FALSE;
-	}
+	i_assert(priv != NULL);
 	if (EVP_PKEY_base_id(priv) != EVP_PKEY_EC) {
 		if (error_r != NULL)
 			*error_r = "Only EC key supported";
@@ -1979,16 +1967,12 @@ bool dcrypt_openssl_public_key_id(struct dcrypt_public_key *key, const char *alg
 	const EVP_MD *md = EVP_get_digestbyname(algorithm);
 	EVP_PKEY *pub = (EVP_PKEY*)key;
 
+	i_assert(pub != NULL);
 	if (md == NULL) {
 		if (error_r != NULL)
 			*error_r = t_strdup_printf("Unknown cipher %s", algorithm);
 		return FALSE;
 	}
-	if (pub == NULL) {
-		if (error_r != NULL)
-			*error_r = "key is NULL";
-		return FALSE;
-	}
 
 	return dcrypt_openssl_public_key_id_evp(pub, md, result, error_r);
 }
@@ -1999,16 +1983,12 @@ bool dcrypt_openssl_private_key_id(struct dcrypt_private_key *key, const char *a
 	const EVP_MD *md = EVP_get_digestbyname(algorithm);
 	EVP_PKEY *priv = (EVP_PKEY*)key;
 
+	i_assert(priv != NULL);
 	if (md == NULL) {
 		if (error_r != NULL)
 			*error_r = t_strdup_printf("Unknown cipher %s", algorithm);
 		return FALSE;
 	}
-	if (priv == NULL) {
-		if (error_r != NULL)
-			*error_r = "key is NULL";
-		return FALSE;
-	}
 
 	return dcrypt_openssl_public_key_id_evp(priv, md, result, error_r);
 }

From 92b22c978606d56558c52d2cc6b9256e916d2e06 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 19 Jun 2016 21:20:27 +0300
Subject: [PATCH 370/450] lib-dcrypt: dcrypt_key_convert_private_to_public()
 can no longer fail.

Removed unnecessary failure handling.
---
 src/lib-dcrypt/dcrypt-openssl.c | 17 ++++++++---------
 src/lib-dcrypt/dcrypt-private.h |  2 +-
 src/lib-dcrypt/dcrypt.c         |  4 ++--
 src/lib-dcrypt/dcrypt.h         |  2 +-
 src/lib-dcrypt/test-crypto.c    |  6 +++---
 5 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index 4e055b4ecda..d67d916cc80 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -106,8 +106,6 @@ struct dcrypt_private_key {
 	void *ctx;
 };
 
-static
-bool dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key, const char **error_r);
 static
 bool dcrypt_openssl_public_key_id(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r);
 static
@@ -117,7 +115,7 @@ bool dcrypt_openssl_private_key_id(struct dcrypt_private_key *key, const char *a
 static
 bool dcrypt_openssl_private_key_id_old(struct dcrypt_private_key *key, buffer_t *result, const char **error_r);
 static
-bool dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r, const char **error_r ATTR_UNUSED);
+void dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r);
 static
 void dcrypt_openssl_free_private_key(struct dcrypt_private_key **key);
 static
@@ -714,7 +712,8 @@ bool dcrypt_openssl_generate_keypair(struct dcrypt_keypair *pair_r, enum dcrypt_
 	if (kind == DCRYPT_KEY_RSA) {
 		if (dcrypt_openssl_generate_rsa_key(bits, &pkey, error_r)) {
 			pair_r->priv = (struct dcrypt_private_key*)pkey;
-			return dcrypt_openssl_private_to_public_key(pair_r->priv, &(pair_r->pub), error_r);
+			dcrypt_openssl_private_to_public_key(pair_r->priv, &(pair_r->pub));
+			return TRUE;
 		} else return dcrypt_openssl_error(error_r);
 	} else if (kind == DCRYPT_KEY_EC) {
 		int nid = OBJ_sn2nid(curve);
@@ -725,7 +724,8 @@ bool dcrypt_openssl_generate_keypair(struct dcrypt_keypair *pair_r, enum dcrypt_
 		}
 		if (dcrypt_openssl_generate_ec_key(nid, &pkey, error_r)) {
 			pair_r->priv = (struct dcrypt_private_key*)pkey;
-			return dcrypt_openssl_private_to_public_key(pair_r->priv, &(pair_r->pub), error_r);
+			dcrypt_openssl_private_to_public_key(pair_r->priv, &(pair_r->pub));
+			return TRUE;
 		} else return dcrypt_openssl_error(error_r);
 	}
 	if (error_r != NULL)
@@ -1021,8 +1021,8 @@ bool dcrypt_openssl_load_private_key_dovecot_v2(struct dcrypt_private_key **key_
 		buffer_t *data = buffer_create_dynamic(pool_datastack_create(), 128);
 
 		/* check that we have correct decryption key */
-		if (!dcrypt_openssl_private_to_public_key(dec_key, &pubkey, error_r) ||
-		    !dcrypt_openssl_public_key_id(pubkey, "sha256", data, error_r)) {
+		dcrypt_openssl_private_to_public_key(dec_key, &pubkey);
+		if (!dcrypt_openssl_public_key_id(pubkey, "sha256", data, error_r)) {
 			if (pubkey != NULL) dcrypt_openssl_free_public_key(&pubkey);
 			return FALSE;
 		}
@@ -1619,7 +1619,7 @@ bool dcrypt_openssl_store_public_key(struct dcrypt_public_key *key, enum dcrypt_
 }
 
 static
-bool dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r, const char **error_r)
+void dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r)
 {
 	EVP_PKEY *pkey = (EVP_PKEY*)priv_key;
 	EVP_PKEY *pk;
@@ -1643,7 +1643,6 @@ bool dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, s
 	}
 
 	*pub_key_r = (struct dcrypt_public_key*)pk;
-	return TRUE;
 }
 
 static
diff --git a/src/lib-dcrypt/dcrypt-private.h b/src/lib-dcrypt/dcrypt-private.h
index c8d1e121e9a..06aa47809c1 100644
--- a/src/lib-dcrypt/dcrypt-private.h
+++ b/src/lib-dcrypt/dcrypt-private.h
@@ -72,7 +72,7 @@ struct dcrypt_vfs {
 		const char *password, struct dcrypt_public_key *enc_key, const char **error_r);
 	bool (*store_public_key)(struct dcrypt_public_key *key, enum dcrypt_key_format format, buffer_t *destination, const char **error_r);
 
-	bool (*private_to_public_key)(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r, const char **error_r);
+	void (*private_to_public_key)(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r);
 
 	bool (*key_string_get_info)(const char *key_data, enum dcrypt_key_format *format_r, enum dcrypt_key_version *version_r,
 		enum dcrypt_key_kind *kind_r, enum dcrypt_key_encryption_type *encryption_type_r, const char **encryption_key_hash_r,
diff --git a/src/lib-dcrypt/dcrypt.c b/src/lib-dcrypt/dcrypt.c
index 76468991fe8..1801ae84797 100644
--- a/src/lib-dcrypt/dcrypt.c
+++ b/src/lib-dcrypt/dcrypt.c
@@ -218,9 +218,9 @@ bool dcrypt_key_store_public(struct dcrypt_public_key *key, enum dcrypt_key_form
 	return dcrypt_vfs->store_public_key(key, format, destination, error_r);
 }
 
-bool dcrypt_key_convert_private_to_public(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r, const char **error_r)
+void dcrypt_key_convert_private_to_public(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r)
 {
-	return dcrypt_vfs->private_to_public_key(priv_key, pub_key_r, error_r);
+	dcrypt_vfs->private_to_public_key(priv_key, pub_key_r);
 }
 bool dcrypt_key_string_get_info(const char *key_data, enum dcrypt_key_format *format_r, enum dcrypt_key_version *version_r,
 	enum dcrypt_key_kind *kind_r, enum dcrypt_key_encryption_type *encryption_type_r, const char **encryption_key_hash_r,
diff --git a/src/lib-dcrypt/dcrypt.h b/src/lib-dcrypt/dcrypt.h
index 94f8b4c2b2c..89e6027dee4 100644
--- a/src/lib-dcrypt/dcrypt.h
+++ b/src/lib-dcrypt/dcrypt.h
@@ -182,7 +182,7 @@ bool dcrypt_key_store_private(struct dcrypt_private_key *key, enum dcrypt_key_fo
 
 bool dcrypt_key_store_public(struct dcrypt_public_key *key, enum dcrypt_key_format format, buffer_t *destination, const char **error_r);
 
-bool dcrypt_key_convert_private_to_public(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r, const char **error_r);
+void dcrypt_key_convert_private_to_public(struct dcrypt_private_key *priv_key, struct dcrypt_public_key **pub_key_r);
 
 void dcrypt_keypair_free(struct dcrypt_keypair *keypair);
 
diff --git a/src/lib-dcrypt/test-crypto.c b/src/lib-dcrypt/test-crypto.c
index 3af434b0885..d1fb75cafed 100644
--- a/src/lib-dcrypt/test-crypto.c
+++ b/src/lib-dcrypt/test-crypto.c
@@ -204,7 +204,7 @@ void test_load_v1_key(void)
 		buffer_set_used_size(key_1, 0);
 		/* check that key_id matches */
 		struct dcrypt_public_key *pubkey = NULL;
-		dcrypt_key_convert_private_to_public(pkey, &pubkey, &error);
+		dcrypt_key_convert_private_to_public(pkey, &pubkey);
 		dcrypt_key_store_public(pubkey, DCRYPT_FORMAT_DOVECOT, key_1, NULL);
 		buffer_set_used_size(key_1, 0);
 		dcrypt_key_id_public(pubkey, "sha256", key_1, &error);
@@ -218,7 +218,7 @@ void test_load_v1_key(void)
 			buffer_set_used_size(key_1, 0);
 			/* check that key_id matches */
 			struct dcrypt_public_key *pubkey = NULL;
-			dcrypt_key_convert_private_to_public(pkey2, &pubkey, &error);
+			dcrypt_key_convert_private_to_public(pkey2, &pubkey);
 			dcrypt_key_store_public(pubkey, DCRYPT_FORMAT_DOVECOT, key_1, NULL);
 			buffer_set_used_size(key_1, 0);
 			dcrypt_key_id_public_old(pubkey, key_1, &error);
@@ -270,7 +270,7 @@ void test_load_v2_key(void)
 	dcrypt_key_free_private(&priv);
 
 	struct dcrypt_public_key *pub = NULL;
-	test_assert_idx(dcrypt_key_convert_private_to_public(priv2, &pub, &error), 3);
+	dcrypt_key_convert_private_to_public(priv2, &pub);
 	test_assert_idx(dcrypt_key_load_private(&priv, DCRYPT_FORMAT_DOVECOT, keys[3], NULL, priv2, &error), 3);
 	test_assert_idx(dcrypt_key_store_private(priv, DCRYPT_FORMAT_DOVECOT, "ecdh-aes-256-ctr", tmp, NULL, pub, &error), 3);
 	buffer_set_used_size(tmp, 0);

From 0e8ca25fa462d251fd5cc6bd53a90b06353b2009 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 19 Jun 2016 21:21:56 +0300
Subject: [PATCH 371/450] lib-dcrypt: dcrypt_key_type_public/private() can no
 longer fail.

Removed unnecessary failure handling.
---
 src/lib-dcrypt/dcrypt-openssl.c  | 18 ++++++++----------
 src/lib-dcrypt/dcrypt-private.h  |  4 ++--
 src/lib-dcrypt/dcrypt.c          |  8 ++++----
 src/lib-dcrypt/dcrypt.h          |  4 ++--
 src/lib-dcrypt/ostream-encrypt.c |  2 +-
 5 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index d67d916cc80..81dcd645163 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -1855,25 +1855,23 @@ bool dcrypt_openssl_name2oid(const char *name, buffer_t *oid, const char **error
 }
 
 static
-bool dcrypt_openssl_private_key_type(struct dcrypt_private_key *key, enum dcrypt_key_type *key_type)
+enum dcrypt_key_type dcrypt_openssl_private_key_type(struct dcrypt_private_key *key)
 {
 	EVP_PKEY *priv = (EVP_PKEY*)key;
-	if (priv == NULL) return FALSE;
-	if (EVP_PKEY_base_id(priv) == EVP_PKEY_RSA) *key_type = DCRYPT_KEY_RSA;
-	else if (EVP_PKEY_base_id(priv) == EVP_PKEY_EC) *key_type = DCRYPT_KEY_EC;
+	i_assert(priv != NULL);
+	if (EVP_PKEY_base_id(priv) == EVP_PKEY_RSA) return DCRYPT_KEY_RSA;
+	else if (EVP_PKEY_base_id(priv) == EVP_PKEY_EC) return DCRYPT_KEY_EC;
 	else i_unreached();
-	return FALSE;
 }
 
 static
-bool dcrypt_openssl_public_key_type(struct dcrypt_public_key *key, enum dcrypt_key_type *key_type)
+enum dcrypt_key_type dcrypt_openssl_public_key_type(struct dcrypt_public_key *key)
 {
 	EVP_PKEY *pub = (EVP_PKEY*)key;
-	if (pub == NULL) return FALSE;
-	if (EVP_PKEY_base_id(pub) == EVP_PKEY_RSA) *key_type = DCRYPT_KEY_RSA;
-	else if (EVP_PKEY_base_id(pub) == EVP_PKEY_EC) *key_type = DCRYPT_KEY_EC;
+	i_assert(pub != NULL);
+	if (EVP_PKEY_base_id(pub) == EVP_PKEY_RSA) return DCRYPT_KEY_RSA;
+	else if (EVP_PKEY_base_id(pub) == EVP_PKEY_EC) return DCRYPT_KEY_EC;
 	else i_unreached();
-	return FALSE;
 }
 
 /** this is the v1 old legacy way of doing key id's **/
diff --git a/src/lib-dcrypt/dcrypt-private.h b/src/lib-dcrypt/dcrypt-private.h
index 06aa47809c1..5e4a60d14c2 100644
--- a/src/lib-dcrypt/dcrypt-private.h
+++ b/src/lib-dcrypt/dcrypt-private.h
@@ -90,8 +90,8 @@ struct dcrypt_vfs {
 	const char *(*oid2name)(const unsigned char *oid, size_t oid_len, const char **error_r);
 	bool (*name2oid)(const char *name, buffer_t *oid, const char **error_r);
 
-	bool (*private_key_type)(struct dcrypt_private_key *key, enum dcrypt_key_type *key_type);
-	bool (*public_key_type)(struct dcrypt_public_key *key, enum dcrypt_key_type *key_type);
+	enum dcrypt_key_type (*private_key_type)(struct dcrypt_private_key *key);
+	enum dcrypt_key_type (*public_key_type)(struct dcrypt_public_key *key);
 	bool (*public_key_id)(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r);
 	bool (*public_key_id_old)(struct dcrypt_public_key *key, buffer_t *result, const char **error_r);
 	bool (*private_key_id)(struct dcrypt_private_key *key, const char *algorithm, buffer_t *result, const char **error_r);
diff --git a/src/lib-dcrypt/dcrypt.c b/src/lib-dcrypt/dcrypt.c
index 1801ae84797..a6845ad4e45 100644
--- a/src/lib-dcrypt/dcrypt.c
+++ b/src/lib-dcrypt/dcrypt.c
@@ -230,13 +230,13 @@ bool dcrypt_key_string_get_info(const char *key_data, enum dcrypt_key_format *fo
 		encryption_key_hash_r, key_hash_r, error_r);
 }
 
-bool dcrypt_key_type_private(struct dcrypt_private_key *key, enum dcrypt_key_type *type)
+enum dcrypt_key_type dcrypt_key_type_private(struct dcrypt_private_key *key)
 {
-	return dcrypt_vfs->private_key_type(key, type);
+	return dcrypt_vfs->private_key_type(key);
 }
-bool dcrypt_key_type_public(struct dcrypt_public_key *key, enum dcrypt_key_type *type)
+enum dcrypt_key_type dcrypt_key_type_public(struct dcrypt_public_key *key)
 {
-	return dcrypt_vfs->public_key_type(key, type);
+	return dcrypt_vfs->public_key_type(key);
 }
 bool dcrypt_key_id_public(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r)
 {
diff --git a/src/lib-dcrypt/dcrypt.h b/src/lib-dcrypt/dcrypt.h
index 89e6027dee4..f6d3e44ed64 100644
--- a/src/lib-dcrypt/dcrypt.h
+++ b/src/lib-dcrypt/dcrypt.h
@@ -189,8 +189,8 @@ void dcrypt_keypair_free(struct dcrypt_keypair *keypair);
 void dcrypt_key_free_public(struct dcrypt_public_key **key);
 void dcrypt_key_free_private(struct dcrypt_private_key **key);
 
-bool dcrypt_key_type_private(struct dcrypt_private_key *key, enum dcrypt_key_type *type);
-bool dcrypt_key_type_public(struct dcrypt_public_key *key, enum dcrypt_key_type *type);
+enum dcrypt_key_type dcrypt_key_type_private(struct dcrypt_private_key *key);
+enum dcrypt_key_type dcrypt_key_type_public(struct dcrypt_public_key *key);
 bool dcrypt_key_id_public(struct dcrypt_public_key *key, const char *algorithm, buffer_t *result, const char **error_r); /* return digest of key */
 bool dcrypt_key_id_public_old(struct dcrypt_public_key *key, buffer_t *result, const char **error_r); /* return SHA1 sum of key */
 bool dcrypt_key_id_private(struct dcrypt_private_key *key, const char *algorithm, buffer_t *result, const char **error_r); /* return digest of key */
diff --git a/src/lib-dcrypt/ostream-encrypt.c b/src/lib-dcrypt/ostream-encrypt.c
index 70e69d3fb7b..0569253a0ca 100644
--- a/src/lib-dcrypt/ostream-encrypt.c
+++ b/src/lib-dcrypt/ostream-encrypt.c
@@ -258,7 +258,7 @@ int o_stream_encrypt_key_for_pubkey_v2(struct encrypt_ostream *stream, const cha
 	encrypted_key = buffer_create_dynamic(pool_datastack_create(), 256);
 	temp_key = buffer_create_dynamic(pool_datastack_create(), 48);
 
-	dcrypt_key_type_public(pubkey, &ktype);
+	ktype = dcrypt_key_type_public(pubkey);
 
 	if (ktype == DCRYPT_KEY_RSA) {
 		/* encrypt key as R (as we don't need DH with RSA)*/

From c39dd401306a25c6448706a04fa37b06fb9e6b60 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 19 Jun 2016 22:18:04 +0300
Subject: [PATCH 372/450] lib-dcrypt: Don't ignore BIO errors.

Might happen due to out of memory?
---
 src/lib-dcrypt/dcrypt-openssl.c | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index 81dcd645163..7f90b3540a6 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -1506,7 +1506,7 @@ bool dcrypt_openssl_load_public_key(struct dcrypt_public_key **key_r, enum dcryp
 	BIO *key_in = BIO_new_mem_buf((void*)data, strlen(data));
 
 	key = PEM_read_bio_PUBKEY(key_in, &key, NULL, NULL);
-	(void)BIO_reset(key_in);
+	if (BIO_reset(key_in) <= 0) i_unreached();
 	if (key == NULL) { /* ec keys are bother */
 		/* read the header */
 		char buf[27]; /* begin public key */
@@ -1565,7 +1565,8 @@ bool dcrypt_openssl_store_private_key(struct dcrypt_private_key *key, enum dcryp
 
 	ec = PEM_write_bio_PrivateKey(key_out, pkey, algo, NULL, 0, NULL, (void*)password);
 
-	(void)BIO_flush(key_out);
+	if (BIO_flush(key_out) <= 0)
+		ec = -1;
 
 	if (ec != 1) {
 		BIO_vfree(key_out);
@@ -1598,10 +1599,12 @@ bool dcrypt_openssl_store_public_key(struct dcrypt_public_key *key, enum dcrypt_
 		(void)BIO_puts(key_out, "-----BEGIN PUBLIC KEY-----\n");
 		(void)BIO_push(b64, key_out);
 		ec = i2d_EC_PUBKEY_bio(b64, EVP_PKEY_get0_EC_KEY(pkey));
-		(void)BIO_flush(b64);
+		if (BIO_flush(b64) <= 0)
+			ec = -1;
 		(void)BIO_pop(b64);
 		BIO_vfree(b64);
-		(void)BIO_puts(key_out, "-----END PUBLIC KEY-----");
+		if (BIO_puts(key_out, "-----END PUBLIC KEY-----") <= 0)
+			ec = -1;
 	}
 
 	if (ec != 1) {

From 15648d9c7687bec4eabed678a812d6a66009e0be Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 19 Jun 2016 22:21:59 +0300
Subject: [PATCH 373/450] lib-dcrypt: Added missing error handling.

Most of these are probably unnecessary now that malloc() no longer fails.
Also some of the NULL checks may not be needed since OpenSSL functions
(usually?) return failure on NULL parameters, but sometimes they perform
a different operation. So overall, probably safer to include these checks.
---
 src/lib-dcrypt/dcrypt-openssl.c | 118 +++++++++++++++++++++++++-------
 1 file changed, 92 insertions(+), 26 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index 7f90b3540a6..8b11bbfdea3 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -465,7 +465,7 @@ bool dcrypt_openssl_ctx_hmac_init(struct dcrypt_context_hmac *ctx, const char **
 	i_assert(ctx->md != NULL);
 #if SSLEAY_VERSION_NUMBER >= 0x1010000fL
 	ctx->ctx = HMAC_CTX_new();
-	if (ctx->ctx == NULL) return FALSE;
+	if (ctx->ctx == NULL) return dcrypt_openssl_error(error_r);
 	ec = HMAC_Init_ex(ctx->ctx, ctx->key, ctx->klen, ctx->md, NULL);
 #else
 	HMAC_CTX_init(&ctx->ctx);
@@ -527,7 +527,8 @@ bool dcrypt_openssl_generate_ec_key(int nid, EVP_PKEY **key, const char **error_
 
 	/* generate key from parameters */
 	ctx = EVP_PKEY_CTX_new(params, NULL);
-	if (EVP_PKEY_keygen_init(ctx) < 1 ||
+	if (ctx == NULL ||
+	    EVP_PKEY_keygen_init(ctx) < 1 ||
 	    EVP_PKEY_keygen(ctx, key) < 1)
 	{
 		dcrypt_openssl_error(error_r);
@@ -569,10 +570,13 @@ bool dcrypt_openssl_ecdh_derive_secret_local(struct dcrypt_private_key *local_ke
 {
 	EVP_PKEY *local = (EVP_PKEY*)local_key;
 	BN_CTX *bn_ctx = BN_CTX_new();
+	if (bn_ctx == NULL)
+		return dcrypt_openssl_error(error_r);
 	const EC_GROUP *grp = EC_KEY_get0_group(EVP_PKEY_get0_EC_KEY(local));
 	EC_POINT *pub = EC_POINT_new(grp);
 	/* convert ephemeral key data EC point */
-	if (EC_POINT_oct2point(grp, pub, R->data, R->used, bn_ctx) != 1)
+	if (pub == NULL ||
+	    EC_POINT_oct2point(grp, pub, R->data, R->used, bn_ctx) != 1)
 	{
 		EC_POINT_free(pub);
 		BN_CTX_free(bn_ctx);
@@ -580,19 +584,27 @@ bool dcrypt_openssl_ecdh_derive_secret_local(struct dcrypt_private_key *local_ke
 	}
 	EC_KEY *ec_key = EC_KEY_new();
 	/* convert point to public key */
-	EC_KEY_set_conv_form(ec_key, POINT_CONVERSION_COMPRESSED);
-	EC_KEY_set_group(ec_key, grp);
-	EC_KEY_set_public_key(ec_key, pub);
+	int ec = 0;
+	if (ec_key == NULL ||
+	    EC_KEY_set_group(ec_key, grp) != 1 ||
+	    EC_KEY_set_public_key(ec_key, pub) != 1)
+		ec = -1;
+	else
+		EC_KEY_set_conv_form(ec_key, POINT_CONVERSION_COMPRESSED);
 	EC_POINT_free(pub);
 	BN_CTX_free(bn_ctx);
 
 	/* make sure it looks like a valid key */
-	if (EC_KEY_check_key(ec_key) != 1) {
+	if (ec == -1 || EC_KEY_check_key(ec_key) != 1) {
 		EC_KEY_free(ec_key);
 		return dcrypt_openssl_error(error_r);
 	}
 
 	EVP_PKEY *peer = EVP_PKEY_new();
+	if (peer == NULL) {
+		EC_KEY_free(ec_key);
+		return dcrypt_openssl_error(error_r);
+	}
 	EVP_PKEY_set1_EC_KEY(peer, ec_key);
 	EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new(local, NULL);
 
@@ -852,7 +864,7 @@ bool dcrypt_openssl_load_private_key_dovecot_v1(struct dcrypt_private_key **key_
 	/* decode and optionally decipher private key value */
 	if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_NONE) {
 		point = BN_new();
-		if (BN_hex2bn(&point, input[3]) < 1) {
+		if (point == NULL || BN_hex2bn(&point, input[3]) < 1) {
 			BN_free(point);
 			EC_KEY_free(eckey);
 			return dcrypt_openssl_error(error_r);
@@ -882,11 +894,20 @@ bool dcrypt_openssl_load_private_key_dovecot_v1(struct dcrypt_private_key **key_
 
 	/* assign private key */
 	BN_CTX *bnctx = BN_CTX_new();
+	if (bnctx == NULL) {
+		EC_KEY_free(eckey);
+		return dcrypt_openssl_error(error_r);
+	}
 	EC_KEY_set_conv_form(eckey, POINT_CONVERSION_COMPRESSED);
 	EC_KEY_set_private_key(eckey, point);
 	EC_KEY_precompute_mult(eckey, bnctx);
 	EC_KEY_set_asn1_flag(eckey, OPENSSL_EC_NAMED_CURVE);
 	EC_POINT *pub = EC_POINT_new(EC_KEY_get0_group(eckey));
+	if (pub == NULL) {
+		EC_KEY_free(eckey);
+		BN_CTX_free(bnctx);
+		return dcrypt_openssl_error(error_r);
+	}
 	/* calculate public key */
 	ec = EC_POINT_mul(EC_KEY_get0_group(eckey), pub, point, NULL, NULL, bnctx);
 	EC_KEY_set_public_key(eckey, pub);
@@ -899,6 +920,10 @@ bool dcrypt_openssl_load_private_key_dovecot_v1(struct dcrypt_private_key **key_
 		unsigned char digest[SHA256_DIGEST_LENGTH];
 		/* validate that the key was loaded correctly */
 		char *id = ec_key_get_pub_point_hex(eckey);
+		if (id == NULL) {
+			EC_KEY_free(eckey);
+			return dcrypt_openssl_error(error_r);
+		}
 		SHA256((unsigned char*)id, strlen(id), digest);
 		OPENSSL_free(id);
 		const char *digest_hex = binary_to_hex(digest, SHA256_DIGEST_LENGTH);
@@ -909,6 +934,10 @@ bool dcrypt_openssl_load_private_key_dovecot_v1(struct dcrypt_private_key **key_
 			return FALSE;
 		}
 		EVP_PKEY *key = EVP_PKEY_new();
+		if (key == NULL) {
+			EC_KEY_free(eckey);
+			return dcrypt_openssl_error(error_r);
+		}
 		EVP_PKEY_set1_EC_KEY(key, eckey);
 		EC_KEY_free(eckey);
 		*key_r = (struct dcrypt_private_key *)key;
@@ -1090,7 +1119,8 @@ bool dcrypt_openssl_load_private_key_dovecot_v2(struct dcrypt_private_key **key_
 	if (EVP_PKEY_type(nid) == EVP_PKEY_RSA) {
 		RSA *rsa = RSA_new();
 		const unsigned char *ptr = buffer_get_data(key_data, NULL);
-		if (d2i_RSAPrivateKey(&rsa, &ptr, key_data->used) == NULL ||
+		if (rsa == NULL ||
+		    d2i_RSAPrivateKey(&rsa, &ptr, key_data->used) == NULL ||
 		    RSA_check_key(rsa) != 1) {
 			safe_memset(buffer_get_modifiable_data(key_data, NULL), 0, key_data->used);
 			RSA_free(rsa);
@@ -1099,12 +1129,17 @@ bool dcrypt_openssl_load_private_key_dovecot_v2(struct dcrypt_private_key **key_
 		safe_memset(buffer_get_modifiable_data(key_data, NULL), 0, key_data->used);
 		buffer_set_used_size(key_data, 0);
 		EVP_PKEY *pkey = EVP_PKEY_new();
+		if (pkey == NULL) {
+			RSA_free(rsa);
+			return dcrypt_openssl_error(error_r);
+		}
 		EVP_PKEY_set1_RSA(pkey, rsa);
 		*key_r = (struct dcrypt_private_key *)pkey;
 	} else {
 		int ec;
 		BIGNUM *point = BN_new();
-		if (BN_mpi2bn(key_data->data, key_data->used, point) == NULL) {
+		if (point == NULL ||
+		    BN_mpi2bn(key_data->data, key_data->used, point) == NULL) {
 			safe_memset(buffer_get_modifiable_data(key_data, NULL), 0, key_data->used);
 			BN_free(point);
 			return dcrypt_openssl_error(error_r);
@@ -1112,28 +1147,36 @@ bool dcrypt_openssl_load_private_key_dovecot_v2(struct dcrypt_private_key **key_
 		EC_KEY *eckey = EC_KEY_new_by_curve_name(nid);
 		safe_memset(buffer_get_modifiable_data(key_data, NULL), 0, key_data->used);
 		buffer_set_used_size(key_data, 0);
-		if (eckey == NULL) {
+		BN_CTX *bnctx = BN_CTX_new();
+		if (eckey == NULL || bnctx == NULL) {
+			BN_free(point);
+			EC_KEY_free(eckey);
+			BN_CTX_free(bnctx);
 			return dcrypt_openssl_error(error_r);
 		}
-		BN_CTX *bnctx = BN_CTX_new();
 		EC_KEY_set_conv_form(eckey, POINT_CONVERSION_COMPRESSED);
 		EC_KEY_set_private_key(eckey, point);
 		EC_KEY_precompute_mult(eckey, bnctx);
 		EC_KEY_set_asn1_flag(eckey, OPENSSL_EC_NAMED_CURVE);
 		EC_POINT *pub = EC_POINT_new(EC_KEY_get0_group(eckey));
-		/* calculate public key */
-		ec = EC_POINT_mul(EC_KEY_get0_group(eckey), pub, point, NULL, NULL, bnctx);
-		EC_KEY_set_public_key(eckey, pub);
+		if (pub == NULL)
+			ec = -1;
+		else {
+			/* calculate public key */
+			ec = EC_POINT_mul(EC_KEY_get0_group(eckey), pub, point, NULL, NULL, bnctx);
+			EC_KEY_set_public_key(eckey, pub);
+			EC_POINT_free(pub);
+		}
 		BN_free(point);
-		EC_POINT_free(pub);
 		BN_CTX_free(bnctx);
 		/* make sure the EC key is valid */
-		if (ec == 1 && EC_KEY_check_key(eckey) == 1) {
-			EVP_PKEY *key = EVP_PKEY_new();
+		EVP_PKEY *key = EVP_PKEY_new();
+		if (ec == 1 && key != NULL && EC_KEY_check_key(eckey) == 1) {
 			EVP_PKEY_set1_EC_KEY(key, eckey);
 			EC_KEY_free(eckey);
 			*key_r = (struct dcrypt_private_key *)key;
 		} else {
+			EVP_PKEY_free(key);
 			EC_KEY_free(eckey);
 			return dcrypt_openssl_error(error_r);
 		}
@@ -1204,7 +1247,8 @@ int dcrypt_openssl_load_public_key_dovecot_v1(struct dcrypt_public_key **key_r,
 	BN_CTX *bnctx = BN_CTX_new();
 
 	EC_POINT *point = EC_POINT_new(EC_KEY_get0_group(eckey));
-	if (EC_POINT_hex2point(EC_KEY_get0_group(eckey),
+	if (bnctx == NULL || point == NULL ||
+	    EC_POINT_hex2point(EC_KEY_get0_group(eckey),
  	    input[2], point, bnctx) == NULL) {
 		BN_CTX_free(bnctx);
 		EC_KEY_free(eckey);
@@ -1248,7 +1292,7 @@ int dcrypt_openssl_load_public_key_dovecot_v2(struct dcrypt_public_key **key_r,
 	ptr = keybuf;
 
 	EVP_PKEY *pkey = EVP_PKEY_new();
-	if (d2i_PUBKEY(&pkey, &ptr, keylen)==NULL) {
+	if (pkey == NULL || d2i_PUBKEY(&pkey, &ptr, keylen)==NULL) {
 		EVP_PKEY_free(pkey);
 		dcrypt_openssl_error(error_r);
 		return -1;
@@ -1504,6 +1548,8 @@ bool dcrypt_openssl_load_public_key(struct dcrypt_public_key **key_r, enum dcryp
 		return dcrypt_openssl_load_public_key_dovecot(key_r, data, error_r);
 
 	BIO *key_in = BIO_new_mem_buf((void*)data, strlen(data));
+	if (key_in == NULL)
+		return dcrypt_openssl_error(error_r);
 
 	key = PEM_read_bio_PUBKEY(key_in, &key, NULL, NULL);
 	if (BIO_reset(key_in) <= 0) i_unreached();
@@ -1520,12 +1566,19 @@ bool dcrypt_openssl_load_public_key(struct dcrypt_public_key **key_r, enum dcryp
 			return FALSE;
 		}
 		BIO *b64 = BIO_new(BIO_f_base64());
+		if (b64 == NULL) {
+			BIO_vfree(key_in);
+			return dcrypt_openssl_error(error_r);
+		}
 		EC_KEY *eckey = d2i_EC_PUBKEY_bio(b64, NULL);
 		if (eckey != NULL) {
 			EC_KEY_set_conv_form(eckey, POINT_CONVERSION_COMPRESSED);
 			EC_KEY_set_asn1_flag(eckey, OPENSSL_EC_NAMED_CURVE);
 			key = EVP_PKEY_new();
-			EVP_PKEY_set1_EC_KEY(key, eckey);
+			if (key == NULL)
+				EC_KEY_free(eckey);
+			else
+				EVP_PKEY_set1_EC_KEY(key, eckey);
 		}
 	}
 
@@ -1553,6 +1606,9 @@ bool dcrypt_openssl_store_private_key(struct dcrypt_private_key *key, enum dcryp
 
 	EVP_PKEY *pkey = (EVP_PKEY*)key;
 	BIO *key_out = BIO_new(BIO_s_mem());
+	if (key_out == NULL)
+		return dcrypt_openssl_error(error_r);
+
 	const EVP_CIPHER *algo = NULL;
 	if (cipher != NULL) {
 		algo = EVP_get_cipherbyname(cipher);
@@ -1591,11 +1647,15 @@ bool dcrypt_openssl_store_public_key(struct dcrypt_public_key *key, enum dcrypt_
 
 	EVP_PKEY *pkey = (EVP_PKEY*)key;
 	BIO *key_out = BIO_new(BIO_s_mem());
+	if (key_out == NULL)
+		return dcrypt_openssl_error(error_r);
 
+	BIO *b64;
 	if (EVP_PKEY_base_id(pkey) == EVP_PKEY_RSA)
 		ec = PEM_write_bio_PUBKEY(key_out, pkey);
+	else if ((b64 = BIO_new(BIO_f_base64())) == NULL)
+		ec = -1;
 	else {
-		BIO *b64 = BIO_new(BIO_f_base64());
 		(void)BIO_puts(key_out, "-----BEGIN PUBLIC KEY-----\n");
 		(void)BIO_push(b64, key_out);
 		ec = i2d_EC_PUBKEY_bio(b64, EVP_PKEY_get0_EC_KEY(pkey));
@@ -1627,9 +1687,10 @@ void dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, s
 	EVP_PKEY *pkey = (EVP_PKEY*)priv_key;
 	EVP_PKEY *pk;
 
-	if (*pub_key_r == NULL)
+	if (*pub_key_r == NULL) {
 		pk = EVP_PKEY_new();
-	else
+		i_assert(pk != NULL); /* we shouldn't get malloc() failures */
+	} else
 		pk = (EVP_PKEY*)*pub_key_r;
 
 	if (EVP_PKEY_base_id(pkey) == EVP_PKEY_RSA)
@@ -1892,6 +1953,8 @@ bool dcrypt_openssl_public_key_id_old(struct dcrypt_public_key *key, buffer_t *r
 	}
 
 	char *pub_pt_hex = ec_key_get_pub_point_hex(EVP_PKEY_get0_EC_KEY(pub));
+	if (pub_pt_hex == NULL)
+		return dcrypt_openssl_error(error_r);
 	/* digest this */
 	SHA256((const unsigned char*)pub_pt_hex, strlen(pub_pt_hex), buf);
 	buffer_append(result, buf, SHA256_DIGEST_LENGTH);
@@ -1913,6 +1976,8 @@ bool dcrypt_openssl_private_key_id_old(struct dcrypt_private_key *key, buffer_t
 	}
 
 	char *pub_pt_hex = ec_key_get_pub_point_hex(EVP_PKEY_get0_EC_KEY(priv));
+	if (pub_pt_hex == NULL)
+		return dcrypt_openssl_error(error_r);
 	/* digest this */
 	SHA256((const unsigned char*)pub_pt_hex, strlen(pub_pt_hex), buf);
 	buffer_append(result, buf, SHA256_DIGEST_LENGTH);
@@ -1931,7 +1996,7 @@ bool dcrypt_openssl_public_key_id_evp(EVP_PKEY *key, const EVP_MD *md, buffer_t
 		EC_KEY_set_conv_form(EVP_PKEY_get0_EC_KEY(key), POINT_CONVERSION_COMPRESSED);
 	}
 	BIO *b = BIO_new(BIO_s_mem());
-	if (i2d_PUBKEY_bio(b, key) < 1) {
+	if (b == NULL || i2d_PUBKEY_bio(b, key) < 1) {
 		BIO_vfree(b);
 		return dcrypt_openssl_error(error_r);
 	}
@@ -1943,7 +2008,8 @@ bool dcrypt_openssl_public_key_id_evp(EVP_PKEY *key, const EVP_MD *md, buffer_t
 #else
 	EVP_MD_CTX *ctx = EVP_MD_CTX_create();
 #endif
-	if (EVP_DigestInit_ex(ctx, md, NULL) < 1 ||
+	if (ctx == NULL ||
+	    EVP_DigestInit_ex(ctx, md, NULL) < 1 ||
 	    EVP_DigestUpdate(ctx, (const unsigned char*)ptr, len) < 1 ||
 	    EVP_DigestFinal_ex(ctx, buf, &hlen) < 1) {
 		res = dcrypt_openssl_error(error_r);

From 7caacdeeea8cc587b8c963af7d6ee067adb8894f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 19 Jun 2016 22:27:00 +0300
Subject: [PATCH 374/450] lib-dcrypt: Check for all the return values in unit
 tests

---
 src/lib-dcrypt/test-crypto.c | 8 ++++----
 src/lib-dcrypt/test-stream.c | 8 ++++----
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/lib-dcrypt/test-crypto.c b/src/lib-dcrypt/test-crypto.c
index d1fb75cafed..afc0c92c4e4 100644
--- a/src/lib-dcrypt/test-crypto.c
+++ b/src/lib-dcrypt/test-crypto.c
@@ -134,7 +134,7 @@ void test_cipher_aead_test_vectors(void)
 	test_assert(dcrypt_ctx_sym_init(ctx, &error));
 	test_assert(dcrypt_ctx_sym_update(ctx, pt->data, pt->used, res, &error));
 	test_assert(dcrypt_ctx_sym_final(ctx, res, &error));
-	dcrypt_ctx_sym_get_tag(ctx, tag_res);
+	test_assert(dcrypt_ctx_sym_get_tag(ctx, tag_res));
 
 	test_assert(buffer_cmp(ct, res) == TRUE);
 	test_assert(buffer_cmp(tag, tag_res) == TRUE);
@@ -205,7 +205,7 @@ void test_load_v1_key(void)
 		/* check that key_id matches */
 		struct dcrypt_public_key *pubkey = NULL;
 		dcrypt_key_convert_private_to_public(pkey, &pubkey);
-		dcrypt_key_store_public(pubkey, DCRYPT_FORMAT_DOVECOT, key_1, NULL);
+		test_assert(dcrypt_key_store_public(pubkey, DCRYPT_FORMAT_DOVECOT, key_1, NULL));
 		buffer_set_used_size(key_1, 0);
 		dcrypt_key_id_public(pubkey, "sha256", key_1, &error);
 		test_assert(strcmp("792caad4d38c9eb2134a0cbc844eae386116de096a0ccafc98479825fc99b6a1", binary_to_hex(key_1->data, key_1->used)) == 0);
@@ -219,9 +219,9 @@ void test_load_v1_key(void)
 			/* check that key_id matches */
 			struct dcrypt_public_key *pubkey = NULL;
 			dcrypt_key_convert_private_to_public(pkey2, &pubkey);
-			dcrypt_key_store_public(pubkey, DCRYPT_FORMAT_DOVECOT, key_1, NULL);
+			test_assert(dcrypt_key_store_public(pubkey, DCRYPT_FORMAT_DOVECOT, key_1, NULL));
 			buffer_set_used_size(key_1, 0);
-			dcrypt_key_id_public_old(pubkey, key_1, &error);
+			test_assert(dcrypt_key_id_public_old(pubkey, key_1, &error));
 			test_assert(strcmp("7c9a1039ea2e4fed73e81dd3ffc3fa22ea4a28352939adde7bf8ea858b00fa4f", binary_to_hex(key_1->data, key_1->used)) == 0);
 
 			dcrypt_key_free_public(&pubkey);
diff --git a/src/lib-dcrypt/test-stream.c b/src/lib-dcrypt/test-stream.c
index 2384ba29246..9031bd1c35a 100644
--- a/src/lib-dcrypt/test-stream.c
+++ b/src/lib-dcrypt/test-stream.c
@@ -220,10 +220,10 @@ int main(void) {
 	dcrypt_initialize("openssl", NULL, NULL);
 	random_init();
 
-	dcrypt_key_load_private(&test_v1_kp.priv, DCRYPT_FORMAT_PEM, key_v1_priv, NULL, NULL, NULL);
-	dcrypt_key_load_public(&test_v1_kp.pub, DCRYPT_FORMAT_PEM, key_v1_pub, NULL);
-	dcrypt_key_load_private(&test_v2_kp.priv, DCRYPT_FORMAT_PEM, key_v2_priv, NULL, NULL, NULL);
-	dcrypt_key_load_public(&test_v2_kp.pub, DCRYPT_FORMAT_PEM, key_v2_pub, NULL);
+	test_assert(dcrypt_key_load_private(&test_v1_kp.priv, DCRYPT_FORMAT_PEM, key_v1_priv, NULL, NULL, NULL));
+	test_assert(dcrypt_key_load_public(&test_v1_kp.pub, DCRYPT_FORMAT_PEM, key_v1_pub, NULL));
+	test_assert(dcrypt_key_load_private(&test_v2_kp.priv, DCRYPT_FORMAT_PEM, key_v2_priv, NULL, NULL, NULL));
+	test_assert(dcrypt_key_load_public(&test_v2_kp.pub, DCRYPT_FORMAT_PEM, key_v2_pub, NULL));
 
 	static void (*test_functions[])(void) = {
 		test_static_v1_input,

From e9ff07194055d1d0794b9afc7520ef8baaa960ec Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 11:33:47 +0300
Subject: [PATCH 375/450] lib-dcrypt: dcrypt_keypair_generate() no longer
 assumes pair_r to be initialized.

It wasn't clear that it should have been zeroed. It also likely isn't very
useful to be able to place the generated key to existing keys.
---
 src/lib-dcrypt/dcrypt.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lib-dcrypt/dcrypt.c b/src/lib-dcrypt/dcrypt.c
index a6845ad4e45..7592aa8a91f 100644
--- a/src/lib-dcrypt/dcrypt.c
+++ b/src/lib-dcrypt/dcrypt.c
@@ -196,6 +196,7 @@ bool dcrypt_pbkdf2(const unsigned char *password, size_t password_len, const uns
 
 bool dcrypt_keypair_generate(struct dcrypt_keypair *pair_r, enum dcrypt_key_type kind, unsigned int bits, const char *curve, const char **error_r)
 {
+	memset(pair_r, 0, sizeof(*pair_r));
 	return dcrypt_vfs->generate_keypair(pair_r, kind, bits, curve, error_r);
 }
 

From cc9df1198822fbe838026dee05ef92d96a969961 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 21 Jun 2016 21:06:22 +0300
Subject: [PATCH 376/450] LAYOUT=index: Avoid unnecessary work for setting
 \Marked flags in LIST reply.

If MAILBOX_LIST_ITER_RETURN_NO_FLAGS is set, the caller doesn't care about
the flags.
---
 src/lib-storage/list/mailbox-list-index-iter.c | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/lib-storage/list/mailbox-list-index-iter.c b/src/lib-storage/list/mailbox-list-index-iter.c
index 08e02827b42..cbaf435032b 100644
--- a/src/lib-storage/list/mailbox-list-index-iter.c
+++ b/src/lib-storage/list/mailbox-list-index-iter.c
@@ -101,10 +101,12 @@ mailbox_list_index_update_info(struct mailbox_list_index_iterate_context *ctx)
 						    &ctx->info.flags);
 	}
 
-	box = mailbox_alloc(ctx->ctx.list, ctx->info.vname, 0);
-	mailbox_list_index_status_set_info_flags(box, node->uid,
-						 &ctx->info.flags);
-	mailbox_free(&box);
+	if ((ctx->ctx.flags & MAILBOX_LIST_ITER_RETURN_NO_FLAGS) == 0) {
+		box = mailbox_alloc(ctx->ctx.list, ctx->info.vname, 0);
+		mailbox_list_index_status_set_info_flags(box, node->uid,
+							 &ctx->info.flags);
+		mailbox_free(&box);
+	}
 }
 
 static void

From 7ffa59270a5ac8a39b2a84436e6077ef2647e8c0 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 21 Jun 2016 21:39:49 +0300
Subject: [PATCH 377/450] lib-storage: Added a kludgy quick-"parameter" to
 list_index_has_changed()

---
 src/lib-storage/list/mailbox-list-index-status.c | 3 +++
 src/lib-storage/mail-storage-private.h           | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/src/lib-storage/list/mailbox-list-index-status.c b/src/lib-storage/list/mailbox-list-index-status.c
index 8b65943dfa8..c495c03d88b 100644
--- a/src/lib-storage/list/mailbox-list-index-status.c
+++ b/src/lib-storage/list/mailbox-list-index-status.c
@@ -786,8 +786,11 @@ void mailbox_list_index_status_set_info_flags(struct mailbox *box, uint32_t uid,
 		/* our in-memory tree is out of sync */
 		ret = 1;
 	} else T_BEGIN {
+		/* kludge: avoid breaking API for v2.2.x. Fixed in v2.3.x. */
+		box->list_index_has_changed_quick = TRUE;
 		ret = box->v.list_index_has_changed == NULL ? 0 :
 			box->v.list_index_has_changed(box, view, seq);
+		box->list_index_has_changed_quick = FALSE;
 	} T_END;
 
 	if (ret != 0) {
diff --git a/src/lib-storage/mail-storage-private.h b/src/lib-storage/mail-storage-private.h
index 4d8bba30972..38946336472 100644
--- a/src/lib-storage/mail-storage-private.h
+++ b/src/lib-storage/mail-storage-private.h
@@ -410,6 +410,8 @@ struct mailbox {
 	unsigned int update_first_saved:1;
 	/* mailbox_verify_create_name() only checks for mailbox_verify_name() */
 	unsigned int skip_create_name_restrictions:1;
+	/* v2.2.x API kludge: quick-parameter to list_index_has_changed() */
+	unsigned int list_index_has_changed_quick:1;
 };
 
 struct mail_vfuncs {

From f194f5c497dd32503701716d6eaa167878495ea8 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 21 Jun 2016 21:26:29 +0300
Subject: [PATCH 378/450] LAYOUT=index: Existence or GUID lookups don't need to
 refresh mailboxes.

---
 .../list/mailbox-list-index-status.c          | 22 ++++++++++++-------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/lib-storage/list/mailbox-list-index-status.c b/src/lib-storage/list/mailbox-list-index-status.c
index c495c03d88b..82f96433f04 100644
--- a/src/lib-storage/list/mailbox-list-index-status.c
+++ b/src/lib-storage/list/mailbox-list-index-status.c
@@ -34,8 +34,8 @@ struct index_list_storage_module index_list_storage_module =
 	((box)->inbox_any)
 
 static int
-index_list_open_view(struct mailbox *box, struct mail_index_view **view_r,
-		     uint32_t *seq_r)
+index_list_open_view(struct mailbox *box, bool refresh,
+		     struct mail_index_view **view_r, uint32_t *seq_r)
 {
 	struct mailbox_list_index *ilist = INDEX_LIST_CONTEXT(box->list);
 	struct mailbox_list_index_node *node;
@@ -58,6 +58,9 @@ index_list_open_view(struct mailbox *box, struct mail_index_view **view_r,
 	if (!mail_index_lookup_seq(view, node->uid, &seq)) {
 		/* our in-memory tree is out of sync */
 		ret = 1;
+	} else if (!refresh) {
+		/* this operation doesn't need the index to be up-to-date */
+		ret = 0;
 	} else T_BEGIN {
 		ret = box->v.list_index_has_changed == NULL ? 0 :
 			box->v.list_index_has_changed(box, view, seq);
@@ -89,7 +92,7 @@ index_list_exists(struct mailbox *box, bool auto_boxes,
 	uint32_t seq;
 	int ret;
 
-	if ((ret = index_list_open_view(box, &view, &seq)) <= 0) {
+	if ((ret = index_list_open_view(box, FALSE, &view, &seq)) <= 0) {
 		/* failure / not found. fallback to the real storage check
 		   just in case to see if the mailbox was just created. */
 		return ibox->module_ctx.super.
@@ -186,6 +189,9 @@ index_list_get_cached_status(struct mailbox *box,
 	uint32_t seq;
 	int ret;
 
+	if (items == 0)
+		return 1;
+
 	if ((items & STATUS_UNSEEN) != 0 &&
 	    (mailbox_get_private_flags_mask(box) & MAIL_SEEN) != 0) {
 		/* can't get UNSEEN from list index, since each user has
@@ -193,7 +199,7 @@ index_list_get_cached_status(struct mailbox *box,
 		return 0;
 	}
 
-	if ((ret = index_list_open_view(box, &view, &seq)) <= 0)
+	if ((ret = index_list_open_view(box, TRUE, &view, &seq)) <= 0)
 		return ret;
 
 	ret = mailbox_list_index_status(box->list, view, seq, items,
@@ -230,7 +236,7 @@ index_list_get_cached_guid(struct mailbox *box, guid_128_t guid_r)
 		return 0;
 	}
 
-	if ((ret = index_list_open_view(box, &view, &seq)) <= 0)
+	if ((ret = index_list_open_view(box, FALSE, &view, &seq)) <= 0)
 		return ret;
 
 	ret = mailbox_list_index_status(box->list, view, seq, 0,
@@ -252,7 +258,7 @@ static int index_list_get_cached_vsize(struct mailbox *box, uoff_t *vsize_r)
 
 	i_assert(!ilist->syncing);
 
-	if ((ret = index_list_open_view(box, &view, &seq)) <= 0)
+	if ((ret = index_list_open_view(box, TRUE, &view, &seq)) <= 0)
 		return ret;
 
 	ret = mailbox_list_index_status(box->list, view, seq,
@@ -283,7 +289,7 @@ index_list_get_cached_first_saved(struct mailbox *box,
 
 	memset(first_saved_r, 0, sizeof(*first_saved_r));
 
-	if ((ret = index_list_open_view(box, &view, &seq)) <= 0)
+	if ((ret = index_list_open_view(box, TRUE, &view, &seq)) <= 0)
 		return ret;
 
 	mail_index_lookup_ext(view, seq, ilist->first_saved_ext_id,
@@ -700,7 +706,7 @@ void mailbox_list_index_update_mailbox_index(struct mailbox *box,
 	int ret;
 
 	memset(&changes, 0, sizeof(changes));
-	if ((ret = index_list_open_view(box, &list_view, &changes.seq)) <= 0)
+	if ((ret = index_list_open_view(box, TRUE, &list_view, &changes.seq)) <= 0)
 		return;
 
 	(void)mailbox_list_index_status(box->list, list_view, changes.seq,

From 064fe96f766796a16a25ede7043e02d0e0abd51c Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 21 Jun 2016 22:08:25 +0300
Subject: [PATCH 379/450] welcome plugin: -Wstrict-bool warning fix

---
 src/plugins/welcome/welcome-plugin.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plugins/welcome/welcome-plugin.c b/src/plugins/welcome/welcome-plugin.c
index 715529351fa..090219c9a2b 100644
--- a/src/plugins/welcome/welcome-plugin.c
+++ b/src/plugins/welcome/welcome-plugin.c
@@ -29,7 +29,7 @@ static void script_execute(struct mail_user *user, const char *cmd, bool wait)
 	int fd, ret;
 
 	if (user->mail_debug)
-		i_debug("welcome: Executing %s (wait=%d)", cmd, wait);
+		i_debug("welcome: Executing %s (wait=%d)", cmd, wait ? 1 : 0);
 
 	args = t_strsplit_spaces(cmd, " ");
 	socket_path = args[0];

From 39b4214499b45aaa9045e6bbf4eb50af9e33d486 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 22 Jun 2016 01:24:42 +0300
Subject: [PATCH 380/450] fts-lucene: Fixed crash on error or auto-rebuild
 conditions.

---
 src/plugins/fts-lucene/lucene-wrapper.cc | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/plugins/fts-lucene/lucene-wrapper.cc b/src/plugins/fts-lucene/lucene-wrapper.cc
index 138b70392ea..3752f891849 100644
--- a/src/plugins/fts-lucene/lucene-wrapper.cc
+++ b/src/plugins/fts-lucene/lucene-wrapper.cc
@@ -820,15 +820,15 @@ rescan_next(struct rescan_context *ctx, Document *doc)
 }
 
 static void
-rescan_clear_unseen_mailbox(struct rescan_context *rescan_ctx,
-			    struct mailbox_list *list,
+rescan_clear_unseen_mailbox(struct lucene_index *index,
+			    struct rescan_context *rescan_ctx,
 			    const char *vname,
 			    const struct fts_index_header *hdr)
 {
 	struct mailbox *box;
 	struct mailbox_metadata metadata;
 
-	box = mailbox_alloc(list, vname,
+	box = mailbox_alloc(index->list, vname,
 			    (enum mailbox_flags)0);
 	if (mailbox_open(box) == 0 &&
 	    mailbox_get_metadata(box, MAILBOX_METADATA_GUID,
@@ -861,7 +861,7 @@ static void rescan_clear_unseen_mailboxes(struct lucene_index *index,
 
 	iter = mailbox_list_iter_init(index->list, "*", iter_flags);
 	while ((info = mailbox_list_iter_next(iter)) != NULL)
-		rescan_clear_unseen_mailbox(rescan_ctx, index->list, info->vname, &hdr);
+		rescan_clear_unseen_mailbox(index, rescan_ctx, info->vname, &hdr);
 	(void)mailbox_list_iter_deinit(&iter);
 
 	if (ns->prefix_len > 0 &&
@@ -869,7 +869,7 @@ static void rescan_clear_unseen_mailboxes(struct lucene_index *index,
 		/* namespace prefix itself isn't returned by the listing */
 		vname = t_strndup(index->list->ns->prefix,
 				  index->list->ns->prefix_len-1);
-		rescan_clear_unseen_mailbox(rescan_ctx, index->list, vname, &hdr);
+		rescan_clear_unseen_mailbox(index, rescan_ctx, vname, &hdr);
 	}
 }
 

From 90d2a4aff8d77897074f9a2c428f81e59737281a Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 23 Jun 2016 17:21:09 +0300
Subject: [PATCH 381/450] lazy_expunge: Optimize checking for last instance
 when moving a mail.

It's never the last instance, so we don't need to even check it.
---
 .../lazy-expunge/lazy-expunge-plugin.c        | 43 ++++++++++++++-----
 1 file changed, 33 insertions(+), 10 deletions(-)

diff --git a/src/plugins/lazy-expunge/lazy-expunge-plugin.c b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
index 0a92de6090d..16efb8ef209 100644
--- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c
+++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
@@ -27,6 +27,11 @@
 #define LAZY_EXPUNGE_MAIL_CONTEXT(obj) \
 	MODULE_CONTEXT(obj, lazy_expunge_mail_module)
 
+struct lazy_expunge_mail {
+	union mail_module_context module_ctx;
+	bool moving;
+};
+
 struct lazy_expunge_mail_user {
 	union mail_user_module_context module_ctx;
 
@@ -267,14 +272,18 @@ static void lazy_expunge_mail_expunge(struct mail *_mail)
 	struct lazy_expunge_mail_user *luser =
 		LAZY_EXPUNGE_USER_CONTEXT(ns->user);
 	struct mail_private *mail = (struct mail_private *)_mail;
-	union mail_module_context *mmail = LAZY_EXPUNGE_MAIL_CONTEXT(mail);
+	struct lazy_expunge_mail *mmail = LAZY_EXPUNGE_MAIL_CONTEXT(mail);
 	struct lazy_expunge_transaction *lt =
 		LAZY_EXPUNGE_CONTEXT(_mail->transaction);
 	struct mail *real_mail;
 	struct mail_save_context *save_ctx;
 	const char *error;
+	bool moving = mmail->moving;
 	int ret;
 
+	/* Clear this in case the mail is used for non-move later on. */
+	mmail->moving = FALSE;
+
 	/* don't copy the mail if we're expunging from lazy_expunge
 	   namespace (even if it's via a virtual mailbox) */
 	if (mail_get_backend_mail(_mail, &real_mail) < 0) {
@@ -282,7 +291,7 @@ static void lazy_expunge_mail_expunge(struct mail *_mail)
 		return;
 	}
 	if (lazy_expunge_is_internal_mailbox(real_mail->box)) {
-		mmail->super.expunge(_mail);
+		mmail->module_ctx.super.expunge(_mail);
 		return;
 	}
 
@@ -290,12 +299,14 @@ static void lazy_expunge_mail_expunge(struct mail *_mail)
 		/* we want to copy only the last instance of the mail to
 		   lazy_expunge namespace. other instances will be expunged
 		   immediately. */
-		if ((ret = lazy_expunge_mail_is_last_instace(_mail)) < 0) {
+		if (moving)
+			ret = 0;
+		else if ((ret = lazy_expunge_mail_is_last_instace(_mail)) < 0) {
 			lazy_expunge_set_error(lt, _mail->box->storage);
 			return;
 		}
 		if (ret == 0) {
-			mmail->super.expunge(_mail);
+			mmail->module_ctx.super.expunge(_mail);
 			return;
 		}
 	}
@@ -327,7 +338,18 @@ static void lazy_expunge_mail_expunge(struct mail *_mail)
 	save_ctx->data.flags &= ~MAIL_DELETED;
 	if (mailbox_copy(&save_ctx, _mail) < 0 && !_mail->expunged)
 		lazy_expunge_set_error(lt, lt->dest_box->storage);
-	mmail->super.expunge(_mail);
+	mmail->module_ctx.super.expunge(_mail);
+}
+
+static int lazy_expunge_copy(struct mail_save_context *ctx, struct mail *_mail)
+{
+	struct mail_private *mail = (struct mail_private *)_mail;
+	union mailbox_module_context *mbox =
+		LAZY_EXPUNGE_CONTEXT(ctx->transaction->box);
+	struct lazy_expunge_mail *mmail = LAZY_EXPUNGE_MAIL_CONTEXT(mail);
+
+	mmail->moving = ctx->moving;
+	return mbox->super.copy(ctx, _mail);
 }
 
 static struct mailbox_transaction_context *
@@ -409,17 +431,17 @@ static void lazy_expunge_mail_allocated(struct mail *_mail)
 		LAZY_EXPUNGE_CONTEXT(_mail->transaction);
 	struct mail_private *mail = (struct mail_private *)_mail;
 	struct mail_vfuncs *v = mail->vlast;
-	union mail_module_context *mmail;
+	struct lazy_expunge_mail *mmail;
 
 	if (lt == NULL)
 		return;
 
-	mmail = p_new(mail->pool, union mail_module_context, 1);
-	mmail->super = *v;
-	mail->vlast = &mmail->super;
+	mmail = p_new(mail->pool, struct lazy_expunge_mail, 1);
+	mmail->module_ctx.super = *v;
+	mail->vlast = &mmail->module_ctx.super;
 
 	v->expunge = lazy_expunge_mail_expunge;
-	MODULE_CONTEXT_SET_SELF(mail, lazy_expunge_mail_module, mmail);
+	MODULE_CONTEXT_SET(mail, lazy_expunge_mail_module, mmail);
 }
 
 static int
@@ -457,6 +479,7 @@ static void lazy_expunge_mailbox_allocated(struct mailbox *box)
 	MODULE_CONTEXT_SET_SELF(box, lazy_expunge_mail_storage_module, mbox);
 
 	if (!lazy_expunge_is_internal_mailbox(box)) {
+		v->copy = lazy_expunge_copy;
 		v->transaction_begin = lazy_expunge_transaction_begin;
 		v->transaction_commit = lazy_expunge_transaction_commit;
 		v->transaction_rollback = lazy_expunge_transaction_rollback;

From 60d99d0114f9cf12aae43d972d805c7b52c72129 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 23 Jun 2016 18:04:40 +0300
Subject: [PATCH 382/450] lib-ssl-iostream: Use ENGINE_set_default()

---
 src/lib-ssl-iostream/dovecot-openssl-common.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/lib-ssl-iostream/dovecot-openssl-common.c b/src/lib-ssl-iostream/dovecot-openssl-common.c
index b76c3ab6e56..2e1b8a03c33 100644
--- a/src/lib-ssl-iostream/dovecot-openssl-common.c
+++ b/src/lib-ssl-iostream/dovecot-openssl-common.c
@@ -100,11 +100,11 @@ int dovecot_openssl_common_global_set_engine(const char *engine,
 		dovecot_openssl_engine = NULL;
 		return -1;
 	}
-	if (ENGINE_set_default_RSA(dovecot_openssl_engine) == 0)
-		i_unreached();
-	if (ENGINE_set_default_DSA(dovecot_openssl_engine) == 0)
-		i_unreached();
-	if (ENGINE_set_default_ciphers(dovecot_openssl_engine) == 0)
-		i_unreached();
+	if (ENGINE_set_default(dovecot_openssl_engine, ENGINE_METHOD_ALL) == 0) {
+		*error_r = t_strdup_printf("ENGINE_set_default(%s) failed", engine);
+		ENGINE_free(dovecot_openssl_engine);
+		dovecot_openssl_engine = NULL;
+		return -1;
+	}
 	return 1;
 }

From 3e85269f59a0bffaf4a25b72e2d48843015e8cf3 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 24 Jun 2016 12:14:32 +0300
Subject: [PATCH 383/450] last-login: Ignore the plugin if last_login_dict
 setting is empty

---
 src/plugins/last-login/last-login-plugin.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plugins/last-login/last-login-plugin.c b/src/plugins/last-login/last-login-plugin.c
index ae3f8c3f657..0b59ab2b093 100644
--- a/src/plugins/last-login/last-login-plugin.c
+++ b/src/plugins/last-login/last-login-plugin.c
@@ -72,7 +72,7 @@ static void last_login_mail_user_created(struct mail_user *user)
 	}
 
 	dict_value = mail_user_plugin_getenv(user, "last_login_dict");
-	if (dict_value == NULL)
+	if (dict_value == NULL || dict_value[0] == '\0')
 		return;
 
 	memset(&set, 0, sizeof(set));

From 9491509b84cd473190fe0bc78a8dcf7e399b9d17 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Mon, 27 Jun 2016 13:19:35 +0300
Subject: [PATCH 384/450] istream-decrypt: Correctly check the header length
 for v1

---
 src/lib-dcrypt/istream-decrypt.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/lib-dcrypt/istream-decrypt.c b/src/lib-dcrypt/istream-decrypt.c
index 87b954dd70d..4ca35880b22 100644
--- a/src/lib-dcrypt/istream-decrypt.c
+++ b/src/lib-dcrypt/istream-decrypt.c
@@ -70,7 +70,8 @@ ssize_t i_stream_decrypt_read_header_v1(struct decrypt_istream *stream,
 	buffer_t *key = buffer_create_dynamic(pool_datastack_create(), 256);
 
 	hdr_len = ((data[0] << 8) | data[1]) + 12;
-	if (mlen < hdr_len) {
+
+	if (mlen < hdr_len - pos) {
 		/* try to read more */
 		return 0;
 	}

From e0d192f4dc113b952a831ba2068556587372c007 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Mon, 27 Jun 2016 13:21:05 +0300
Subject: [PATCH 385/450] istream-decrypt: Ensure we can open short v1 files

---
 src/lib-dcrypt/sample-v1_short.asc |  4 ++++
 src/lib-dcrypt/test-stream.c       | 37 ++++++++++++++++++++++++++++++
 2 files changed, 41 insertions(+)
 create mode 100644 src/lib-dcrypt/sample-v1_short.asc

diff --git a/src/lib-dcrypt/sample-v1_short.asc b/src/lib-dcrypt/sample-v1_short.asc
new file mode 100644
index 00000000000..4bc4c06d2a7
--- /dev/null
+++ b/src/lib-dcrypt/sample-v1_short.asc
@@ -0,0 +1,4 @@
+Q1JZUFRFRAMHAQCtAEMDAA+5INzRNr5OAicv3XI4jh//ufjZN9yYr7mElHKNGY+D
+D2ziqHhPKVra6JzBzZvfySntDnDvdLAomafpDVlESMlkACB8mhA56i5P7XPoHdP/
+w/oi6kooNSk5rd57+OqFiwD6TwAgu4C6eZWV+Spojlk8eOAw784ySgMHK8gDrXhA
+Jwg34GcAIPX4RmqXh+7QTAQWtGNHQvjqSygBxSUYkQ3KfPLMymKvAACMLy6/
diff --git a/src/lib-dcrypt/test-stream.c b/src/lib-dcrypt/test-stream.c
index 9031bd1c35a..19f4b59a53b 100644
--- a/src/lib-dcrypt/test-stream.c
+++ b/src/lib-dcrypt/test-stream.c
@@ -41,6 +41,7 @@ static const char key_v2_pub[] = "-----BEGIN PUBLIC KEY-----\n" \
 "-----END PUBLIC KEY-----";
 
 static const char test_sample_v1_hash[] = "1d7cc2cc1f1983f76241cc42389911e88590ad58cf9d54cafeb5b198d3723dd1";
+static const char test_sample_v1_short_hash[] = "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c";
 static const char test_sample_v2_hash[] = "2e31218656dd34db65b321688bf418dee4ee785e99eb9c21e0d29b4af27a863e";
 
 static struct dcrypt_keypair test_v1_kp;
@@ -81,6 +82,41 @@ void test_static_v1_input(void)
 	test_end();
 }
 
+static
+void test_static_v1_input_short(void)
+{
+	ssize_t siz;
+	const struct hash_method *hash = hash_method_lookup("sha256");
+	unsigned char hash_ctx[hash->context_size];
+	unsigned char hash_dgst[hash->digest_size];
+	hash->init(hash_ctx);
+
+	test_begin("test_static_v1_input_short");
+
+	struct istream *is_1 = i_stream_create_file(DCRYPT_SRC_DIR"/sample-v1_short.asc", IO_BLOCK_SIZE);
+	struct istream *is_2 = i_stream_create_base64_decoder(is_1);
+	i_stream_unref(&is_1);
+	struct istream *is_3 = i_stream_create_decrypt(is_2, test_v1_kp.priv);
+	i_stream_unref(&is_2);
+	struct istream *is_4 = i_stream_create_hash(is_3, hash, hash_ctx);
+	i_stream_unref(&is_3);
+
+	while((siz = i_stream_read(is_4))>0) { i_stream_skip(is_4, siz); }
+
+	if (is_4->stream_errno != 0)
+		i_debug("error: %s", i_stream_get_error(is_4));
+
+	test_assert(is_4->stream_errno == 0);
+
+	i_stream_unref(&is_4);
+
+	hash->result(hash_ctx, hash_dgst);
+
+	test_assert(strcmp(test_sample_v1_short_hash, binary_to_hex(hash_dgst, sizeof(hash_dgst))) == 0);
+
+	test_end();
+}
+
 static
 void test_static_v2_input(void)
 {
@@ -227,6 +263,7 @@ int main(void) {
 
 	static void (*test_functions[])(void) = {
 		test_static_v1_input,
+		test_static_v1_input_short,
 		test_static_v2_input,
 		test_write_read_v1,
 		test_write_read_v2,

From e91e99359e4116f39fb920dbfc1928c17b5b4500 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 27 Jun 2016 14:13:15 +0300
Subject: [PATCH 386/450] lib-dcrypt: Added sample-v1_short.asc to EXTRA_DIST

---
 src/lib-dcrypt/Makefile.am | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lib-dcrypt/Makefile.am b/src/lib-dcrypt/Makefile.am
index b51e774ab4a..946590e7b74 100644
--- a/src/lib-dcrypt/Makefile.am
+++ b/src/lib-dcrypt/Makefile.am
@@ -36,6 +36,7 @@ pkginc_lib_HEADERS = $(headers)
 
 EXTRA_DIST = \
 	sample-v1.asc \
+	sample-v1_short.asc \
 	sample-v2.asc
 
 test_programs = test-crypto test-stream

From eddbff81e95ccc2e8cb638a12be7250aefc7e249 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Mon, 27 Jun 2016 14:43:27 +0300
Subject: [PATCH 387/450] dcrypt-openssl: Various fixes

Fix v1 and v2 key handling and some allocation issues.
---
 src/lib-dcrypt/dcrypt-openssl.c | 80 +++++++++++++++++++++++----------
 1 file changed, 57 insertions(+), 23 deletions(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index 8b11bbfdea3..6447a46c439 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -55,7 +55,7 @@
 
  public key
  ----------
- 2<tab>HEX(i2d_PUBKEY)
+ 2<tab>HEX(i2d_PUBKEY)<tab>key id
 
  - enctype none
  2<tab>key algo oid<tab>0<tab>(RSA = i2d_PrivateKey, EC=Private Point)<tab>key id
@@ -549,6 +549,7 @@ bool dcrypt_openssl_generate_ec_key(int nid, EVP_PKEY **key, const char **error_
 static
 bool dcrypt_openssl_generate_rsa_key(int bits, EVP_PKEY **key, const char **error_r)
 {
+	i_assert(bits >= 256);
 	int ec = 0;
 
 	EVP_PKEY_CTX *ctx;
@@ -568,6 +569,7 @@ bool dcrypt_openssl_generate_rsa_key(int bits, EVP_PKEY **key, const char **erro
 static
 bool dcrypt_openssl_ecdh_derive_secret_local(struct dcrypt_private_key *local_key, buffer_t *R, buffer_t *S, const char **error_r)
 {
+	i_assert(local_key != NULL);
 	EVP_PKEY *local = (EVP_PKEY*)local_key;
 	BN_CTX *bn_ctx = BN_CTX_new();
 	if (bn_ctx == NULL)
@@ -606,6 +608,7 @@ bool dcrypt_openssl_ecdh_derive_secret_local(struct dcrypt_private_key *local_ke
 		return dcrypt_openssl_error(error_r);
 	}
 	EVP_PKEY_set1_EC_KEY(peer, ec_key);
+	EC_KEY_free(ec_key);
 	EVP_PKEY_CTX *pctx = EVP_PKEY_CTX_new(local, NULL);
 
 	/* initialize derivation */
@@ -613,7 +616,7 @@ bool dcrypt_openssl_ecdh_derive_secret_local(struct dcrypt_private_key *local_ke
 	    EVP_PKEY_derive_init(pctx) != 1 ||
 	    EVP_PKEY_derive_set_peer(pctx, peer) != 1) {
 		EVP_PKEY_CTX_free(pctx);
-		EC_KEY_free(ec_key);
+		EVP_PKEY_free(peer);
 		return dcrypt_openssl_error(error_r);
 	}
 
@@ -621,19 +624,18 @@ bool dcrypt_openssl_ecdh_derive_secret_local(struct dcrypt_private_key *local_ke
 	size_t len;
 	if (EVP_PKEY_derive(pctx, NULL, &len) != 1) {
 		EVP_PKEY_CTX_free(pctx);
-		EC_KEY_free(ec_key);
+		EVP_PKEY_free(peer);
 		return dcrypt_openssl_error(error_r);
 	}
 	unsigned char buf[len];
 	memset(buf,0,len);
 	if (EVP_PKEY_derive(pctx, buf, &len) != 1) {
 		EVP_PKEY_CTX_free(pctx);
-		EC_KEY_free(ec_key);
+		EVP_PKEY_free(peer);
 		return dcrypt_openssl_error(error_r);
 	}
 	EVP_PKEY_CTX_free(pctx);
 	buffer_append(S, buf, len);
-	EC_KEY_free(ec_key);
 	EVP_PKEY_free(peer);
 	return TRUE;
 }
@@ -843,7 +845,6 @@ bool dcrypt_openssl_load_private_key_dovecot_v1(struct dcrypt_private_key **key_
 	const char **error_r)
 {
 	int nid, ec, enctype;
-	EC_KEY *eckey = NULL;
 	BIGNUM *point = NULL;
 
 	if (str_to_int(input[1], &nid) != 0) {
@@ -858,15 +859,11 @@ bool dcrypt_openssl_load_private_key_dovecot_v1(struct dcrypt_private_key **key_
 		return FALSE;
 	}
 
-	eckey = EC_KEY_new_by_curve_name(nid);
-	if (eckey == NULL) return dcrypt_openssl_error(error_r);
-
 	/* decode and optionally decipher private key value */
 	if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_NONE) {
 		point = BN_new();
 		if (point == NULL || BN_hex2bn(&point, input[3]) < 1) {
 			BN_free(point);
-			EC_KEY_free(eckey);
 			return dcrypt_openssl_error(error_r);
 		}
 	} else if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PASSWORD) {
@@ -874,24 +871,25 @@ bool dcrypt_openssl_load_private_key_dovecot_v1(struct dcrypt_private_key **key_
 		const char *enc_priv_pt = input[3];
 		const char *salt = input[4];
 		if (!dcrypt_openssl_decrypt_point_password_v1(enc_priv_pt, password, salt, &point, error_r)) {
-			EC_KEY_free(eckey);
 			return FALSE;
 		}
 	} else if (enctype == DCRYPT_DOVECOT_KEY_ENCRYPT_PK) {
 		/* by key */
 		const char *enc_priv_pt = input[3];
 		const char *peer_key = input[4];
+		i_assert(dec_key != NULL);
 		if (!dcrypt_openssl_decrypt_point_ec_v1(dec_key, enc_priv_pt, peer_key, &point, error_r)) {
-			EC_KEY_free(eckey);
 			return FALSE;
 		}
 	} else {
 		if (error_r != NULL)
 			*error_r = "Invalid key data";
-		EC_KEY_free(eckey);
 		return FALSE;
 	}
 
+	EC_KEY *eckey = EC_KEY_new_by_curve_name(nid);
+	if (eckey == NULL) return dcrypt_openssl_error(error_r);
+
 	/* assign private key */
 	BN_CTX *bnctx = BN_CTX_new();
 	if (bnctx == NULL) {
@@ -1134,6 +1132,7 @@ bool dcrypt_openssl_load_private_key_dovecot_v2(struct dcrypt_private_key **key_
 			return dcrypt_openssl_error(error_r);
 		}
 		EVP_PKEY_set1_RSA(pkey, rsa);
+		RSA_free(rsa);
 		*key_r = (struct dcrypt_private_key *)pkey;
 	} else {
 		int ec;
@@ -1226,7 +1225,7 @@ int dcrypt_openssl_load_public_key_dovecot_v1(struct dcrypt_public_key **key_r,
 	int len, const char **input, const char **error_r)
 {
 	int nid;
-	if (len != 3) {
+	if (len != 4) {
 		if (error_r != NULL)
 			*error_r = "Corrupted data";
 		return -1;
@@ -1249,7 +1248,7 @@ int dcrypt_openssl_load_public_key_dovecot_v1(struct dcrypt_public_key **key_r,
 	EC_POINT *point = EC_POINT_new(EC_KEY_get0_group(eckey));
 	if (bnctx == NULL || point == NULL ||
 	    EC_POINT_hex2point(EC_KEY_get0_group(eckey),
- 	    input[2], point, bnctx) == NULL) {
+	    input[2], point, bnctx) == NULL) {
 		BN_CTX_free(bnctx);
 		EC_KEY_free(eckey);
 		EC_POINT_free(point);
@@ -1266,6 +1265,16 @@ int dcrypt_openssl_load_public_key_dovecot_v1(struct dcrypt_public_key **key_r,
 	if (EC_KEY_check_key(eckey) == 1) {
 		EVP_PKEY *key = EVP_PKEY_new();
 		EVP_PKEY_set1_EC_KEY(key, eckey);
+		EC_KEY_free(eckey);
+		/* make sure digest matches */
+		buffer_t *dgst = buffer_create_dynamic(pool_datastack_create(), 32);
+		dcrypt_openssl_public_key_id_old((struct dcrypt_public_key *)key, dgst, NULL);
+		if (strcmp(binary_to_hex(dgst->data, dgst->used), input[len-1]) != 0) {
+			if (error_r != NULL)
+				*error_r = "Key id mismatch after load";
+			EVP_PKEY_free(key);
+			return -1;
+		}
 		*key_r = (struct dcrypt_public_key *)key;
 		return 0;
 	}
@@ -1278,7 +1287,7 @@ static
 int dcrypt_openssl_load_public_key_dovecot_v2(struct dcrypt_public_key **key_r,
 	int len, const char **input, const char **error_r)
 {
-	if (len != 2 || strlen(input[1]) < 2 || (strlen(input[1])%2) != 0) {
+	if (len != 3 || strlen(input[1]) < 2 || (strlen(input[1])%2) != 0) {
 		if (error_r != NULL)
 			*error_r = "Corrupted data";
 		return -1;
@@ -1298,6 +1307,16 @@ int dcrypt_openssl_load_public_key_dovecot_v2(struct dcrypt_public_key **key_r,
 		return -1;
 	}
 
+	/* make sure digest matches */
+	buffer_t *dgst = buffer_create_dynamic(pool_datastack_create(), 32);
+	dcrypt_openssl_public_key_id((struct dcrypt_public_key *)pkey, "sha256", dgst, NULL);
+	if (strcmp(binary_to_hex(dgst->data, dgst->used), input[len-1]) != 0) {
+		if (error_r != NULL)
+			*error_r = "Key id mismatch after load";
+		EVP_PKEY_free(pkey);
+		return -1;
+	}
+
 	*key_r = (struct dcrypt_public_key *)pkey;
 	return 0;
 }
@@ -1494,17 +1513,31 @@ bool dcrypt_openssl_store_public_key_dovecot(struct dcrypt_public_key *key, buff
 {
 	EVP_PKEY *pubkey = (EVP_PKEY*)key;
 	unsigned char *tmp = NULL;
+	size_t dest_used = buffer_get_used_size(destination);
 
 	int rv = i2d_PUBKEY(pubkey, &tmp);
 
 	if (tmp == NULL)
 		return dcrypt_openssl_error(error_r);
+
 	/* then store it */
 	str_append_c(destination, '2');
 	str_append_c(destination, '\t');
 	binary_to_hex_append(destination, tmp, rv);
 	OPENSSL_free(tmp);
 
+	/* append public key ID */
+	str_append_c(destination, '\t');
+
+	buffer_t *buf = buffer_create_dynamic(pool_datastack_create(), 32);
+	bool res = dcrypt_openssl_public_key_id(key, "sha256", buf, error_r);
+
+	if (!res) {
+		buffer_set_used_size(destination, dest_used);
+		return FALSE;
+	}
+
+	str_append(destination, binary_to_hex(buf->data, buf->used));
 	return TRUE;
 }
 
@@ -1575,10 +1608,9 @@ bool dcrypt_openssl_load_public_key(struct dcrypt_public_key **key_r, enum dcryp
 			EC_KEY_set_conv_form(eckey, POINT_CONVERSION_COMPRESSED);
 			EC_KEY_set_asn1_flag(eckey, OPENSSL_EC_NAMED_CURVE);
 			key = EVP_PKEY_new();
-			if (key == NULL)
-				EC_KEY_free(eckey);
-			else
+			if (key != NULL)
 				EVP_PKEY_set1_EC_KEY(key, eckey);
+			EC_KEY_free(eckey);
 		}
 	}
 
@@ -1695,7 +1727,9 @@ void dcrypt_openssl_private_to_public_key(struct dcrypt_private_key *priv_key, s
 
 	if (EVP_PKEY_base_id(pkey) == EVP_PKEY_RSA)
 	{
-		EVP_PKEY_set1_RSA(pk, RSAPublicKey_dup(EVP_PKEY_get0_RSA(pkey)));
+		RSA *rsa = RSAPublicKey_dup(EVP_PKEY_get0_RSA(pkey));
+		EVP_PKEY_set1_RSA(pk, rsa);
+		RSA_free(rsa);
 	} else if (EVP_PKEY_base_id(pkey) == EVP_PKEY_EC) {
 		EC_KEY* eck = EVP_PKEY_get1_EC_KEY(pkey);
 		EC_KEY_set_asn1_flag(eck, OPENSSL_EC_NAMED_CURVE);
@@ -1756,7 +1790,7 @@ bool dcrypt_openssl_key_string_get_info(const char *key_data, enum dcrypt_key_fo
 		/* field 1 - version */
 		if (strcmp(fields[0], "1") == 0) {
 			version = DCRYPT_KEY_VERSION_1;
-			if (nfields == 3) {
+			if (nfields == 4) {
 				kind = DCRYPT_KEY_KIND_PUBLIC;
 			} else if (nfields == 5 && strcmp(fields[2],"0") == 0) {
 				kind = DCRYPT_KEY_KIND_PRIVATE;
@@ -1764,7 +1798,7 @@ bool dcrypt_openssl_key_string_get_info(const char *key_data, enum dcrypt_key_fo
 			} else if (nfields == 6 && strcmp(fields[2],"2") == 0) {
 				kind = DCRYPT_KEY_KIND_PRIVATE;
 				encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_PASSWORD;
-			} else if (nfields == 11 && strcmp(fields[2],"1") == 0) {
+			} else if (nfields == 7 && strcmp(fields[2],"1") == 0) {
 				kind = DCRYPT_KEY_KIND_PRIVATE;
 				encryption_type = DCRYPT_KEY_ENCRYPTION_TYPE_KEY;
 				if (encryption_key_hash_r != NULL)
@@ -1776,7 +1810,7 @@ bool dcrypt_openssl_key_string_get_info(const char *key_data, enum dcrypt_key_fo
 			}
 		} else if (strcmp(fields[0], "2") == 0) {
 			version = DCRYPT_KEY_VERSION_1;
-			if (nfields == 2) {
+			if (nfields == 3) {
 				kind = DCRYPT_KEY_KIND_PUBLIC;
 			} else if (nfields == 5 && strcmp(fields[2],"0") == 0) {
 				kind = DCRYPT_KEY_KIND_PRIVATE;

From 714e6326e25d0473a7f7c6d203b82d4305641a2f Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Mon, 27 Jun 2016 14:43:58 +0300
Subject: [PATCH 388/450] dcrypt: Add tests for v1 and v2 public keys and RSA

---
 src/lib-dcrypt/test-crypto.c | 223 +++++++++++++++++++++++++++++++++--
 1 file changed, 213 insertions(+), 10 deletions(-)

diff --git a/src/lib-dcrypt/test-crypto.c b/src/lib-dcrypt/test-crypto.c
index afc0c92c4e4..64434a5d414 100644
--- a/src/lib-dcrypt/test-crypto.c
+++ b/src/lib-dcrypt/test-crypto.c
@@ -166,6 +166,8 @@ void test_cipher_aead_test_vectors(void)
 static
 void test_hmac_test_vectors(void)
 {
+	test_begin("test_hmac_test_vectors");
+
 	buffer_t *pt, *ct, *key, *res;
 	pt = buffer_create_dynamic(pool_datastack_create(), 50);
 	key = buffer_create_dynamic(pool_datastack_create(), 20);
@@ -187,6 +189,94 @@ void test_hmac_test_vectors(void)
 		test_assert(buffer_cmp(ct, res));
 		dcrypt_ctx_hmac_destroy(&hctx);
 	}
+
+	test_end();
+}
+
+static
+void test_load_v1_keys(void)
+{
+	test_begin("test_load_v1_keys");
+
+	const char *error = NULL;
+	const char *data1 = "1\t716\t1\t0567e6bf9579813ae967314423b0fceb14bda24749303923de9a9bb9370e0026f995901a57e63113eeb2baf0c940e978d00686cbb52bd5014bc318563375876255\t0300E46DA2125427BE968EB3B649910CDC4C405E5FFDE18D433A97CABFEE28CEEFAE9EE356C792004FFB80981D67E741B8CC036A34235A8D2E1F98D1658CFC963D07EB\td0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0\t7c9a1039ea2e4fed73e81dd3ffc3fa22ea4a28352939adde7bf8ea858b00fa4f";
+
+	enum dcrypt_key_format format;
+	enum dcrypt_key_version version;
+	enum dcrypt_key_kind kind;
+	enum dcrypt_key_encryption_type encryption_type;
+	const char *encryption_key_hash = NULL;
+	const char *key_hash = NULL;
+
+	bool ret = dcrypt_key_string_get_info(data1, &format, &version,
+			&kind, &encryption_type, &encryption_key_hash,
+			&key_hash, &error);
+
+	test_assert(ret == TRUE);
+	test_assert(error == NULL);
+	test_assert(format == DCRYPT_FORMAT_DOVECOT);
+	test_assert(version == DCRYPT_KEY_VERSION_1);
+	test_assert(kind == DCRYPT_KEY_KIND_PRIVATE);
+	test_assert(encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_KEY);
+	test_assert(strcmp(encryption_key_hash, "d0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0") == 0);
+	test_assert(strcmp(key_hash, "7c9a1039ea2e4fed73e81dd3ffc3fa22ea4a28352939adde7bf8ea858b00fa4f") == 0);
+
+	const char* data2 = "1\t716\t0301EB00973C4EFC8FCECA4EA33E941F50B561199A5159BCB6C2EED9DD1D62D65E38A254979D89E28F0C28883E71EE2AD264CD16B863FA094A8F6F69A56B62E8918040\t7c9a1039ea2e4fed73e81dd3ffc3fa22ea4a28352939adde7bf8ea858b00fa4f";
+
+	error = NULL;
+	encryption_key_hash = NULL;
+	key_hash = NULL;
+
+	ret = dcrypt_key_string_get_info(data2, &format, &version,
+			&kind, &encryption_type, &encryption_key_hash,
+			&key_hash, &error);
+
+	test_assert(ret == TRUE);
+	test_assert(error == NULL);
+	test_assert(format == DCRYPT_FORMAT_DOVECOT);
+	test_assert(version == DCRYPT_KEY_VERSION_1);
+	test_assert(kind == DCRYPT_KEY_KIND_PUBLIC);
+	test_assert(encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_NONE);
+	test_assert(encryption_key_hash == NULL);
+	test_assert(strcmp(key_hash, "7c9a1039ea2e4fed73e81dd3ffc3fa22ea4a28352939adde7bf8ea858b00fa4f") == 0);
+
+	/* This is the key that should be able to decrypt key1 */
+	const char *data3 = "1\t716\t0\t048FD04FD3612B22D32790C592CF21CEF417EFD2EA34AE5F688FA5B51BED29E05A308B68DA78E16E90B47A11E133BD9A208A2894FD01B0BEE865CE339EA3FB17AC\td0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0";
+
+	error = NULL;
+	encryption_key_hash = NULL;
+	key_hash = NULL;
+
+	ret = dcrypt_key_string_get_info(data3, &format, &version,
+			&kind, &encryption_type, &encryption_key_hash,
+			&key_hash, &error);
+	test_assert(ret == TRUE);
+	test_assert(error == NULL);
+	test_assert(format == DCRYPT_FORMAT_DOVECOT);
+	test_assert(version == DCRYPT_KEY_VERSION_1);
+	test_assert(kind == DCRYPT_KEY_KIND_PRIVATE);
+	test_assert(encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_NONE);
+	test_assert(encryption_key_hash == NULL);
+	test_assert(strcmp(key_hash, "d0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0") == 0);
+
+	/* key3's key_hash should and does match key1's encryption_key_hash */
+	struct dcrypt_private_key *pkey = NULL;
+	struct dcrypt_private_key *pkey2 = NULL;
+	pkey = NULL;
+	error = NULL;
+
+	ret = dcrypt_key_load_private(&pkey2, format, data3, NULL, NULL, &error);
+	test_assert(ret == TRUE);
+	test_assert(error == NULL);
+
+	ret = dcrypt_key_load_private(&pkey, format, data1, NULL, pkey2, &error);
+	test_assert(ret == TRUE);
+	test_assert(error == NULL);
+
+	dcrypt_key_free_private(&pkey2);
+	dcrypt_key_free_private(&pkey);
+
+	test_end();
 }
 
 static
@@ -196,7 +286,7 @@ void test_load_v1_key(void)
 
 	buffer_t *key_1 = buffer_create_dynamic(pool_datastack_create(), 128);
 
-	struct dcrypt_private_key *pkey, *pkey2;
+	struct dcrypt_private_key *pkey = NULL, *pkey2 = NULL;
 	const char *error = NULL;
 
 	test_assert(dcrypt_key_load_private(&pkey, DCRYPT_FORMAT_DOVECOT, "1\t716\t0\t048FD04FD3612B22D32790C592CF21CEF417EFD2EA34AE5F688FA5B51BED29E05A308B68DA78E16E90B47A11E133BD9A208A2894FD01B0BEE865CE339EA3FB17AC\td0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0", NULL, NULL, &error));
@@ -233,6 +323,49 @@ void test_load_v1_key(void)
 	test_end();
 }
 
+static
+void test_load_v1_public_key(void)
+{
+	test_begin("test_load_v1_public_key");
+
+	const char* data1 = "1\t716\t030131D8A5FD5167947A0AE9CB112ADED6526654635AA5887051EE2364414B60FF32EBA8FA0BBE9485DBDE8794BBBCB44BBFC0D662A4287A848BA570D4E5E45A11FE0F\td0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0";
+
+	const char* error = NULL;
+	const char* key_hash = NULL;
+	const char* encryption_key_hash = NULL;
+
+	enum dcrypt_key_format format;
+	enum dcrypt_key_version version;
+	enum dcrypt_key_kind kind;
+	enum dcrypt_key_encryption_type encryption_type;
+
+	bool ret = dcrypt_key_string_get_info(data1, &format, &version,
+			&kind, &encryption_type, &encryption_key_hash,
+			&key_hash, &error);
+
+	test_assert(ret == TRUE);
+	test_assert(error == NULL);
+	test_assert(format == DCRYPT_FORMAT_DOVECOT);
+	test_assert(version == DCRYPT_KEY_VERSION_1);
+	test_assert(kind == DCRYPT_KEY_KIND_PUBLIC);
+	test_assert(encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_NONE);
+	test_assert(key_hash != NULL);
+	test_assert(strcmp(key_hash, "d0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0") == 0);
+	test_assert(encryption_key_hash == NULL);
+
+	struct dcrypt_public_key *pub_key = NULL;
+	ret = dcrypt_key_load_public(&pub_key, format, data1, &error);
+	test_assert(ret == TRUE);
+	test_assert(error == NULL);
+
+	test_assert(dcrypt_key_type_public(pub_key) == DCRYPT_KEY_EC);
+
+	dcrypt_key_free_public(&pub_key);
+	test_assert(pub_key == NULL);
+
+	test_end();
+}
+
 static
 void test_load_v2_key(void)
 {
@@ -288,41 +421,111 @@ void test_load_v2_key(void)
 static
 void test_load_v2_public_key(void)
 {
-	struct dcrypt_public_key *pub;
+	struct dcrypt_public_key *pub = NULL;
 	const char *error;
 
 	test_begin("test_load_v2_public_key");
-	const char *key = "2\t3058301006072a8648ce3d020106052b810400230344000301c50954e734dd8b410a607764a7057065a45510da52f2c6e28e0cb353b9c389fa8cb786943ae991fce9befed78fb162fbbc615415f06af06c8cc80c37f4e94ff6c7";
+	const char *key = "2\t3058301006072a8648ce3d020106052b810400230344000301c50954e734dd8b410a607764a7057065a45510da52f2c6e28e0cb353b9c389fa8cb786943ae991fce9befed78fb162fbbc615415f06af06c8cc80c37f4e94ff6c7\t185a7212542782e239111f9c19d126ad55b18ddaf4883d66afe8d9627c3607d8";
 
 	test_assert(dcrypt_key_load_public(&pub, DCRYPT_FORMAT_DOVECOT, key, &error));
 
 	buffer_t *tmp = buffer_create_dynamic(default_pool, 256);
 
-	test_assert(dcrypt_key_store_public(pub, DCRYPT_FORMAT_DOVECOT, tmp, &error));
+	if (pub != NULL) {
+		test_assert(dcrypt_key_store_public(pub, DCRYPT_FORMAT_DOVECOT, tmp, &error));
+		test_assert(strcmp(key, str_c(tmp))==0);
+		buffer_free(&tmp);
+		dcrypt_key_free_public(&pub);
+	}
 
-	test_assert(strcmp(key, str_c(tmp))==0);
-	buffer_free(&tmp);
-	dcrypt_key_free_public(&pub);
+	test_end();
+}
+
+static
+void test_gen_and_get_info_rsa_pem(void)
+{
+	test_begin("test_gen_and_get_info_rsa_pem");
+
+	const char *error = NULL;
+	bool ret = FALSE;
+	struct dcrypt_keypair pair;
+	string_t* buf = str_new(default_pool, 4096);
+
+	ret = dcrypt_keypair_generate(&pair, DCRYPT_KEY_RSA, 1024, NULL, NULL);
+	test_assert(ret == TRUE);
+
+	/* test public key */
+	enum dcrypt_key_format format;
+	enum dcrypt_key_version version;
+	enum dcrypt_key_kind kind;
+	enum dcrypt_key_encryption_type encryption_type;
+	const char *encryption_key_hash;
+	const char *key_hash;
+
+	ret = dcrypt_key_store_public(pair.pub, DCRYPT_FORMAT_PEM, buf,
+			&error);
+	test_assert(ret == TRUE);
+
+	ret = dcrypt_key_string_get_info(str_c(buf), &format, &version,
+			&kind, &encryption_type, &encryption_key_hash,
+			&key_hash, &error);
+	test_assert(ret == TRUE);
+	test_assert(format == DCRYPT_FORMAT_PEM);
+	test_assert(version == DCRYPT_KEY_VERSION_NA);
+
+	test_assert(kind == DCRYPT_KEY_KIND_PUBLIC);
+	test_assert(encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_NONE);
+	test_assert(encryption_key_hash == NULL);
+	test_assert(key_hash == NULL);
+
+	/* test private key */
+	buffer_set_used_size(buf, 0);
+	ret = dcrypt_key_store_private(pair.priv, DCRYPT_FORMAT_PEM, NULL,
+			buf, NULL, NULL, &error);
+
+	test_assert(ret == TRUE);
+
+	ret = dcrypt_key_string_get_info(str_c(buf), &format, &version,
+			&kind, &encryption_type, &encryption_key_hash,
+			&key_hash, &error);
+
+	test_assert(ret == TRUE);
+	test_assert(format == DCRYPT_FORMAT_PEM);
+	test_assert(version == DCRYPT_KEY_VERSION_NA);
+
+	test_assert(kind == DCRYPT_KEY_KIND_PRIVATE);
+
+	test_assert(encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_NONE);
+	test_assert(encryption_key_hash == NULL);
+	test_assert(key_hash == NULL);
+
+	dcrypt_keypair_free(&pair);
+	buffer_free(&buf);
 
 	test_end();
 }
 
 int main(void) {
-	dcrypt_initialize("openssl", NULL, NULL);
 	random_init();
+	dcrypt_initialize("openssl", NULL, NULL);
+
 	static void (*test_functions[])(void) = {
 		test_cipher_test_vectors,
 		test_cipher_aead_test_vectors,
 		test_hmac_test_vectors,
+		test_load_v1_keys,
 		test_load_v1_key,
+		test_load_v1_public_key,
 		test_load_v2_key,
 		test_load_v2_public_key,
+		test_gen_and_get_info_rsa_pem,
 		NULL
 	};
 
-	int ret;
+	int ret = test_run(test_functions);
 
-	ret = test_run(test_functions);
+	dcrypt_deinitialize();
+	random_deinit();
 
 	return ret;
 }

From 3412a5bda73ce28a1fb2f2821741e8566a6b64cd Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 27 Jun 2016 16:37:18 +0300
Subject: [PATCH 389/450] lib-dcrypt: Make static analyzer happier

---
 src/lib-dcrypt/test-crypto.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/lib-dcrypt/test-crypto.c b/src/lib-dcrypt/test-crypto.c
index 64434a5d414..a6d96b10b79 100644
--- a/src/lib-dcrypt/test-crypto.c
+++ b/src/lib-dcrypt/test-crypto.c
@@ -349,8 +349,7 @@ void test_load_v1_public_key(void)
 	test_assert(version == DCRYPT_KEY_VERSION_1);
 	test_assert(kind == DCRYPT_KEY_KIND_PUBLIC);
 	test_assert(encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_NONE);
-	test_assert(key_hash != NULL);
-	test_assert(strcmp(key_hash, "d0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0") == 0);
+	test_assert(key_hash != NULL && strcmp(key_hash, "d0cfaca5d335f9edc41c84bb47465184cb0e2ec3931bebfcea4dd433615e77a0") == 0);
 	test_assert(encryption_key_hash == NULL);
 
 	struct dcrypt_public_key *pub_key = NULL;

From 8edd4308b6e1da2fe52573bc0becadf98874aca1 Mon Sep 17 00:00:00 2001
From: Baofeng Wang <baofeng.wang@dovecot.fi>
Date: Fri, 10 Jun 2016 14:31:00 +0300
Subject: [PATCH 390/450] fts plugin: Added fts_autoindex_exclude settings.

fts_autoindex_exclude setting specifies special-use flag or mailbox name to
be excluded.

 - If a name starts with '\', it's treated as a case-insensitive special-use
flag.

 - Multiple names can be specified with serial numbers, for example:

  plugin {
    fts_autoindex_exclude = \Junk
    fts_autoindex_exclude2 = \Trash
    fts_autoindex_exclude3 = DUMPSTER
    fts_autoindex_exclude4 = New folder
  }
---
 src/plugins/fts/fts-storage.c | 59 ++++++++++++++++++++++++++++++++++-
 1 file changed, 58 insertions(+), 1 deletion(-)

diff --git a/src/plugins/fts/fts-storage.c b/src/plugins/fts/fts-storage.c
index b4b500982d1..592ec7aa731 100644
--- a/src/plugins/fts/fts-storage.c
+++ b/src/plugins/fts/fts-storage.c
@@ -6,6 +6,7 @@
 #include "str.h"
 #include "strescape.h"
 #include "write-full.h"
+#include "wildcard-match.h"
 #include "mail-search-build.h"
 #include "mail-storage-private.h"
 #include "mailbox-list-private.h"
@@ -39,6 +40,7 @@ struct fts_mailbox_list {
 struct fts_mailbox {
 	union mailbox_module_context module_ctx;
 	struct fts_backend_update_context *sync_update_ctx;
+	bool fts_mailbox_excluded;
 };
 
 struct fts_transaction_context {
@@ -629,7 +631,7 @@ fts_transaction_commit(struct mailbox_transaction_context *t,
 	bool autoindex;
 	int ret = 0;
 
-	autoindex = ft->mails_saved &&
+	autoindex = ft->mails_saved && !fbox->fts_mailbox_excluded &&
 		mail_user_plugin_getenv(box->storage->user,
 					"fts_autoindex") != NULL;
 
@@ -726,6 +728,60 @@ static int fts_copy(struct mail_save_context *ctx, struct mail *mail)
 	return 0;
 }
 
+static const char *const *fts_exclude_get_patterns(struct mail_user *user)
+{
+	ARRAY_TYPE(const_string) patterns;
+	const char *str;
+	char set_name[21+MAX_INT_STRLEN+1];
+	unsigned int i;
+
+	str = mail_user_plugin_getenv(user, "fts_autoindex_exclude");
+	if (str == NULL)
+		return NULL;
+
+	t_array_init(&patterns, 16);
+	for (i = 2; str != NULL; i++) {
+		array_append(&patterns, &str, 1);
+
+		if (i_snprintf(set_name, sizeof(set_name),
+			       "fts_autoindex_exclude%u", i) < 0)
+			i_unreached();
+		str = mail_user_plugin_getenv(user, set_name);
+	}
+	array_append_zero(&patterns);
+	return array_idx(&patterns, 0);
+}
+
+static bool fts_autoindex_exclude_match(struct mailbox *box)
+{
+	const char *const *exclude_list;
+	unsigned int i;
+	const struct mailbox_settings *set;
+	const char *const *special_use;
+	struct mail_user *user = box->storage->user;
+
+	exclude_list = fts_exclude_get_patterns(user);
+	if (exclude_list == NULL)
+		return TRUE;
+
+	set = mailbox_settings_find(mailbox_get_namespace(box),
+				    mailbox_get_vname(box));
+	special_use = set == NULL ? NULL :
+		t_strsplit_spaces(set->special_use, " ");
+	for (i = 0; exclude_list[i] != NULL; i++) {
+		if (exclude_list[i][0] == '\\') {
+			/* \Special-use flag */
+			if (str_array_icase_find(special_use, exclude_list[i]))
+				return TRUE;
+		} else {
+			/* mailbox name with wildcards */
+			if (wildcard_match(box->name, exclude_list[i]))
+				return TRUE;
+		}
+	}
+	return FALSE;
+}
+
 void fts_mailbox_allocated(struct mailbox *box)
 {
 	struct fts_mailbox_list *flist = FTS_LIST_CONTEXT(box->list);
@@ -738,6 +794,7 @@ void fts_mailbox_allocated(struct mailbox *box)
 	fbox = p_new(box->pool, struct fts_mailbox, 1);
 	fbox->module_ctx.super = *v;
 	box->vlast = &fbox->module_ctx.super;
+	fbox->fts_mailbox_excluded = fts_autoindex_exclude_match(box);
 
 	v->get_status = fts_mailbox_get_status;
 	v->search_init = fts_mailbox_search_init;

From 5bbf5616f6aba60f3f8f21d1363c5cee66e5e279 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Fri, 3 Jun 2016 20:21:42 +0300
Subject: [PATCH 391/450] auth-policy: Add policy implementation

---
 src/auth/Makefile.am               |   3 +
 src/auth/auth-request-var-expand.c |   2 +-
 src/auth/auth-request-var-expand.h |   1 +
 src/auth/auth-request.h            |   2 +
 src/auth/auth-settings.c           |  38 ++
 src/auth/auth-settings.h           |   9 +
 src/auth/policy.c                  | 535 +++++++++++++++++++++++++++++
 src/auth/policy.h                  |   8 +
 src/config/settings-get.pl         |   1 +
 9 files changed, 598 insertions(+), 1 deletion(-)
 create mode 100755 src/auth/policy.c
 create mode 100644 src/auth/policy.h

diff --git a/src/auth/Makefile.am b/src/auth/Makefile.am
index 574b854899a..19c974f8a59 100644
--- a/src/auth/Makefile.am
+++ b/src/auth/Makefile.am
@@ -27,6 +27,7 @@ AM_CPPFLAGS = \
 	-I$(top_srcdir)/src/lib-test \
 	-I$(top_srcdir)/src/lib-dict \
 	-I$(top_srcdir)/src/lib-dns \
+	-I$(top_srcdir)/src/lib-http \
 	-I$(top_srcdir)/src/lib-sql \
 	-I$(top_srcdir)/src/lib-settings \
 	-I$(top_srcdir)/src/lib-stats \
@@ -117,6 +118,7 @@ auth_SOURCES = \
 	passdb-sql.c \
 	passdb-static.c \
 	passdb-template.c \
+	policy.c \
 	userdb.c \
 	userdb-blocking.c \
 	userdb-checkpassword.c \
@@ -163,6 +165,7 @@ headers = \
 	passdb-cache.h \
 	passdb-template.h \
 	password-scheme.h \
+	policy.h \
 	userdb.h \
 	userdb-blocking.h \
 	userdb-template.h \
diff --git a/src/auth/auth-request-var-expand.c b/src/auth/auth-request-var-expand.c
index f1f041e473b..7c6ee1c3766 100644
--- a/src/auth/auth-request-var-expand.c
+++ b/src/auth/auth-request-var-expand.c
@@ -157,7 +157,6 @@ auth_request_get_var_expand_table_full(const struct auth_request *auth_request,
 	tab[29].value = strchr(orig_user, '@');
 	if (tab[29].value != NULL)
 		tab[29].value = escape_func(tab[29].value+1, auth_request);
-
 	if (auth_request->master_user != NULL)
 		auth_user = auth_request->master_user;
 	else
@@ -167,6 +166,7 @@ auth_request_get_var_expand_table_full(const struct auth_request *auth_request,
 	tab[32].value = strchr(auth_user, '@');
 	if (tab[32].value != NULL)
 		tab[32].value = escape_func(tab[32].value+1, auth_request);
+
 	return ret_tab;
 }
 
diff --git a/src/auth/auth-request-var-expand.h b/src/auth/auth-request-var-expand.h
index 1362ec3f861..2b0c2aa4d86 100644
--- a/src/auth/auth-request-var-expand.h
+++ b/src/auth/auth-request-var-expand.h
@@ -9,6 +9,7 @@ auth_request_escape_func_t(const char *string,
 #define AUTH_REQUEST_VAR_TAB_USERNAME_IDX 1
 #define AUTH_REQUEST_VAR_TAB_DOMAIN_IDX 2
 #define AUTH_REQUEST_VAR_TAB_COUNT 33
+
 extern const struct var_expand_table
 auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT+1];
 
diff --git a/src/auth/auth-request.h b/src/auth/auth-request.h
index d2778ef4ef8..989d2c79279 100644
--- a/src/auth/auth-request.h
+++ b/src/auth/auth-request.h
@@ -78,6 +78,7 @@ struct auth_request {
 	in_port_t local_port, remote_port, real_local_port, real_remote_port;
 
 	struct timeout *to_abort, *to_penalty;
+	unsigned int policy_penalty;
 	unsigned int last_penalty;
 	unsigned int initial_response_len;
 	const unsigned char *initial_response;
@@ -143,6 +144,7 @@ struct auth_request {
 	   will work. */
 	unsigned int userdb_prefetch_set:1;
 	unsigned int stats_sent:1;
+	unsigned int policy_refusal:1;
 
 	/* ... mechanism specific data ... */
 };
diff --git a/src/auth/auth-settings.c b/src/auth/auth-settings.c
index e5f6c76131c..c942819c1fb 100644
--- a/src/auth/auth-settings.c
+++ b/src/auth/auth-settings.c
@@ -2,6 +2,7 @@
 
 #include "lib.h"
 #include "array.h"
+#include "hash-method.h"
 #include "settings-parser.h"
 #include "master-service-private.h"
 #include "master-service-settings.h"
@@ -236,6 +237,15 @@ static const struct setting_define auth_setting_defines[] = {
 	DEF(SET_STR, proxy_self),
 	DEF(SET_TIME, failure_delay),
 
+	DEF(SET_STR, policy_server_url),
+	DEF(SET_STR, policy_server_api_header),
+	DEF(SET_UINT, policy_server_timeout_msecs),
+	DEF(SET_STR, policy_hash_mech),
+	DEF(SET_STR, policy_hash_nonce),
+	DEF(SET_STR, policy_request_attributes),
+	DEF(SET_BOOL, policy_reject_on_fail),
+	DEF(SET_UINT, policy_hash_truncate),
+
 	DEF(SET_BOOL, stats),
 	DEF(SET_BOOL, verbose),
 	DEF(SET_BOOL, debug),
@@ -276,6 +286,15 @@ static const struct auth_settings auth_default_settings = {
 	.proxy_self = "",
 	.failure_delay = 2,
 
+	.policy_server_url = "",
+	.policy_server_api_header = "",
+	.policy_server_timeout_msecs = 2000,
+	.policy_hash_mech = "sha256",
+	.policy_hash_nonce = "",
+	.policy_request_attributes = "login=%{orig_username} pwhash=%{hashed_password} remote=%{real_rip}",
+	.policy_reject_on_fail = FALSE,
+	.policy_hash_truncate = 12,
+
 	.stats = FALSE,
 	.verbose = FALSE,
 	.debug = FALSE,
@@ -418,6 +437,25 @@ static bool auth_settings_check(void *_set, pool_t pool,
 	set->realms_arr =
 		(const char *const *)p_strsplit_spaces(pool, set->realms, " ");
 
+	if (*set->policy_server_url != '\0') {
+		if (*set->policy_hash_nonce == '\0') {
+
+			*error_r = "auth_policy_hash_nonce must be set when policy server is used";
+			return FALSE;
+		}
+		const struct hash_method *digest = hash_method_lookup(set->policy_hash_mech);
+		if (digest == NULL) {
+			*error_r = "invalid auth_policy_hash_mech given";
+			return FALSE;
+		}
+		if (set->policy_hash_truncate > 0 && set->policy_hash_truncate >= digest->digest_size*8) {
+			*error_r = t_strdup_printf("policy_hash_truncate is not smaller than digest size (%u >= %u)",
+				set->policy_hash_truncate,
+				digest->digest_size*8);
+			return FALSE;
+		}
+	}
+
 	if (!auth_settings_set_self_ips(set, pool, error_r))
 		return FALSE;
 	return TRUE;
diff --git a/src/auth/auth-settings.h b/src/auth/auth-settings.h
index 295420fa0af..1313576a978 100644
--- a/src/auth/auth-settings.h
+++ b/src/auth/auth-settings.h
@@ -53,6 +53,15 @@ struct auth_settings {
 	const char *proxy_self;
 	unsigned int failure_delay;
 
+	const char *policy_server_url;
+	const char *policy_server_api_header;
+	unsigned int policy_server_timeout_msecs;
+	const char *policy_hash_mech;
+	const char *policy_hash_nonce;
+	const char *policy_request_attributes;
+	bool policy_reject_on_fail;
+	unsigned int policy_hash_truncate;
+
 	bool stats;
 	bool verbose, debug, debug_passwords;
 	const char *verbose_passwords;
diff --git a/src/auth/policy.c b/src/auth/policy.c
new file mode 100755
index 00000000000..cbbd6e889e7
--- /dev/null
+++ b/src/auth/policy.c
@@ -0,0 +1,535 @@
+#include "lib.h"
+#include "net.h"
+#include "str.h"
+#include "istream.h"
+#include "ioloop.h"
+#include "base64.h"
+#include "hex-binary.h"
+#include "hash-method.h"
+#include "http-url.h"
+#include "http-client.h"
+#include "json-parser.h"
+#include "auth-request.h"
+#include "auth-penalty.h"
+#include "auth-settings.h"
+#include "policy.h"
+
+#define AUTH_POLICY_DNS_SOCKET_PATH "dns-client"
+
+static struct http_client_settings http_client_set = {
+	.dns_client_socket_path = AUTH_POLICY_DNS_SOCKET_PATH,
+	.max_connect_attempts = 1,
+	.max_idle_time_msecs = 10000,
+	.max_parallel_connections = 100,
+	.debug = 0,
+	.user_agent = "dovecot/auth-policy-client"
+};
+
+static char *auth_policy_json_template;
+
+static struct http_client *http_client;
+
+struct policy_lookup_ctx {
+	pool_t pool;
+	string_t *json;
+	struct auth_request *request;
+	struct http_client_request *http_request;
+	struct json_parser *parser;
+	struct auth_settings *set;
+	const char *url;
+	bool expect_result;
+	int result;
+	const char *message;
+	auth_policy_callback_t callback;
+	void *callback_context;
+
+	struct istream *payload;
+	struct io *io;
+
+	enum {
+		POLICY_RESULT = 0,
+		POLICY_RESULT_VALUE_STATUS,
+		POLICY_RESULT_VALUE_MESSAGE
+	} parse_state;
+
+	bool parse_error;
+};
+
+struct policy_template_keyvalue {
+	const char *key;
+	const char *value;
+};
+
+static
+int auth_policy_attribute_comparator(const struct policy_template_keyvalue *a,
+	const struct policy_template_keyvalue *b)
+{
+	return strcmp(a->key, b->key);
+}
+
+static
+int auth_policy_strptrcmp(const char *a0, const char *a1,
+			  const char *b0, const char *b1)
+{
+	i_assert(a0 <= a1 && b0 <= b1);
+	return memcmp(a0, b0, I_MIN((a1-a0),(b1-b0)));
+}
+
+static
+void auth_policy_open_key(const char *key, string_t *template)
+{
+	const char *ptr;
+	while((ptr = strchr(key, '/')) != NULL) {
+		str_append_c(template,'"');
+		json_append_escaped(template, t_strndup(key, (ptr-key)));
+		str_append_c(template,'"');
+		str_append_c(template,':');
+		str_append_c(template,'{');
+		key = ptr+1;
+	}
+}
+
+static
+void auth_policy_close_key(const char *key, string_t *template)
+{
+	while((key = strchr(key, '/')) != NULL) { str_append_c(template,'}'); key++; }
+}
+
+static
+void auth_policy_open_and_close_to_key(const char *fromkey, const char *tokey, string_t *template)
+{
+	const char *fptr,*tptr,*fdash,*tdash;
+
+	fptr = strrchr(fromkey, '/');
+	tptr = strrchr(tokey, '/');
+
+	if (fptr == NULL && tptr == NULL) return; /* nothing to do */
+
+	if (fptr == NULL && tptr != NULL) {
+		auth_policy_open_key(tokey, template);
+		return;
+	}
+
+	if (fptr != NULL && tptr == NULL) {
+		str_truncate(template, str_len(template)-1);
+
+		auth_policy_close_key(fromkey, template);
+		str_append_c(template, ',');
+		return;
+	}
+
+	if (auth_policy_strptrcmp(fromkey, fptr, tokey, tptr) == 0) {
+		/* nothing to do, again */
+		return;
+	}
+
+	fptr = fromkey;
+	tptr = tokey;
+
+	while(fptr && tptr) {
+		fdash = strchr(fptr, '/');
+		tdash = strchr(tptr, '/');
+
+		if (fdash == NULL) {
+			auth_policy_open_key(tptr, template);
+			break;
+		}
+		if (tdash == NULL) {
+			str_truncate(template, str_len(template)-1);
+			auth_policy_close_key(fptr, template);
+			str_append_c(template, ',');
+			break;
+		}
+		if (auth_policy_strptrcmp(fptr, fdash, tptr, tdash) != 0) {
+			str_truncate(template, str_len(template)-1);
+			auth_policy_close_key(fptr, template);
+			str_append_c(template, ',');
+			auth_policy_open_key(tptr, template);
+			break;
+		}
+		fptr = fdash+1;
+		tptr = tdash+1;
+	}
+}
+
+void auth_policy_init(void)
+{
+	http_client_set.request_absolute_timeout_msecs = global_auth_settings->policy_server_timeout_msecs;
+	if (global_auth_settings->debug)
+		http_client_set.debug = 1;
+	http_client = http_client_init(&http_client_set);
+
+	/* prepare template */
+
+	ARRAY(struct policy_template_keyvalue) attribute_pairs;
+	const struct policy_template_keyvalue *kvptr;
+	string_t *template = t_str_new(64);
+	const char **ptr;
+	const char *key = NULL;
+	const char **list = t_strsplit_spaces(global_auth_settings->policy_request_attributes, "= ");
+
+	t_array_init(&attribute_pairs, 8);
+	for(ptr = list; *ptr != NULL; ptr++) {
+		struct policy_template_keyvalue pair;
+		if (key == NULL) {
+			key = *ptr;
+		} else {
+			pair.key = key;
+			pair.value = *ptr;
+			key = NULL;
+			array_append(&attribute_pairs, &pair, 1);
+		}
+	}
+	if (key != NULL) {
+		i_fatal("auth_policy_request_attributes contains invalid value");
+	}
+
+	/* then we sort it */
+	array_sort(&attribute_pairs, auth_policy_attribute_comparator);
+
+	/* and build a template string */
+	const char *prevkey = "";
+
+	array_foreach(&attribute_pairs, kvptr) {
+		const char *kptr = strchr(kvptr->key, '/');
+		auth_policy_open_and_close_to_key(prevkey, kvptr->key, template);
+		str_append_c(template,'"');
+		json_append_escaped(template, (kptr != NULL?kptr+1:kvptr->key));
+		str_append_c(template,'"');
+		str_append_c(template,':');
+		str_append_c(template,'"');
+		str_append(template,kvptr->value);
+		str_append_c(template,'"');
+		str_append_c(template,',');
+		prevkey = kvptr->key;
+	}
+
+	auth_policy_open_and_close_to_key(prevkey, "", template);
+	str_truncate(template, str_len(template)-1);
+	auth_policy_json_template = i_strdup(str_c(template));
+}
+
+void auth_policy_deinit(void)
+{
+	if (http_client != NULL)
+		http_client_deinit(&http_client);
+	i_free(auth_policy_json_template);
+}
+
+static
+void auth_policy_finish(void *ctx)
+{
+	struct policy_lookup_ctx *context = ctx;
+
+	if (context->parser != NULL) {
+		const char *error ATTR_UNUSED;
+		json_parser_deinit(&(context->parser), &error);
+	}
+	if (context->http_request != NULL)
+		http_client_request_abort(&(context->http_request));
+}
+
+static
+void auth_policy_parse_response(struct policy_lookup_ctx *context)
+{
+	enum json_type type;
+	const char *value;
+	int ret;
+
+	while((ret = json_parse_next(context->parser, &type, &value)) == 1) {
+		if (context->parse_state == POLICY_RESULT) {
+			if (type != JSON_TYPE_OBJECT_KEY)
+				break;
+			else if (strcmp(value, "status") == 0)
+				context->parse_state = POLICY_RESULT_VALUE_STATUS;
+			else if (strcmp(value, "msg") == 0)
+				context->parse_state = POLICY_RESULT_VALUE_MESSAGE;
+			else break;
+		} else if (context->parse_state == POLICY_RESULT_VALUE_STATUS) {
+			if (type != JSON_TYPE_NUMBER || str_to_int(value, &(context->result)) != 0)
+				break;
+			context->parse_state = POLICY_RESULT;
+		} else if (context->parse_state == POLICY_RESULT_VALUE_MESSAGE) {
+			if (type != JSON_TYPE_STRING)
+				break;
+			if (*value != '\0')
+				context->message = p_strdup(context->pool, value);
+			context->parse_state = POLICY_RESULT;
+		} else {
+			break;
+		}
+	}
+
+	if (ret == 0 && !context->payload->eof)
+		return;
+
+	context->parse_error = TRUE;
+
+	io_remove(&(context->io));
+
+	if (context->payload->stream_errno != 0) {
+		auth_request_log_error(context->request, "policy",
+			"Error reading policy server result: %s",
+			i_stream_get_error(context->payload));
+	} else if (ret == 0 && context->payload->eof) {
+		auth_request_log_error(context->request, "policy",
+			"Policy server result was too short");
+	} else if (ret == 1) {
+		auth_request_log_error(context->request, "policy",
+			"Policy server response was malformed");
+	} else {
+		const char *error = "unknown";
+		if (json_parser_deinit(&(context->parser), &error) != 0)
+			auth_request_log_error(context->request, "policy",
+				"Policy server response JSON parse error: %s", error);
+		else if (context->parse_state == POLICY_RESULT)
+			context->parse_error = FALSE;
+	}
+	i_stream_unref(&(context->payload));
+
+	if (context->parse_error) {
+		context->result = (context->set->policy_reject_on_fail ? -1 : 0);
+	}
+
+	context->request->policy_refusal = FALSE;
+
+	if (context->result < 0) {
+		if (context->message != NULL) {
+			/* set message here */
+			auth_request_log_debug(context->request, "policy",
+				"Policy response %d with message: %s",
+				context->result, context->message);
+			auth_request_set_field(context->request, "reason", context->message, NULL);
+		}
+		context->request->policy_refusal = TRUE;
+	} else {
+		auth_request_log_debug(context->request, "policy",
+			"Policy response %d", context->result);
+	}
+
+	if (context->request->policy_refusal == TRUE && context->set->verbose == TRUE) {
+		auth_request_log_info(context->request, "policy", "Authentication failure due to policy server refusal%s%s",
+			(context->message!=NULL?": ":""),
+			(context->message!=NULL?context->message:""));
+	}
+
+	if (context->callback) {
+		context->callback(context->result, context->callback_context);
+	}
+};
+
+static
+void auth_policy_process_response(const struct http_response *response,
+	void *ctx)
+{
+	struct policy_lookup_ctx *context = ctx;
+
+	context->payload = response->payload;
+
+	if ((response->status / 10) != 20) {
+		auth_request_log_error(context->request, "policy",
+			"Policy server HTTP error: %d %s", response->status, response->reason);
+		if (context->callback)
+			context->callback(context->result, context->callback_context);
+		return;
+	}
+
+	if (response->payload == NULL) {
+		if (context->expect_result)
+			auth_request_log_error(context->request, "policy",
+				"Policy server result was empty");
+		if (context->callback)
+			context->callback(context->result, context->callback_context);
+		return;
+	}
+
+	if (context->expect_result) {
+		i_stream_ref(response->payload);
+		context->io = io_add_istream(response->payload, auth_policy_parse_response, context);
+		context->parser = json_parser_init(response->payload);
+		auth_policy_parse_response(ctx);
+	} else {
+		auth_request_log_debug(context->request, "policy",
+			"Policy response %d", context->result);
+		if (context->callback)
+			context->callback(context->result, context->callback_context);
+	}
+}
+
+static
+void auth_policy_send_request(struct policy_lookup_ctx *context)
+{
+	const char *error;
+	struct http_url *url;
+	if (http_url_parse(context->url, NULL, HTTP_URL_ALLOW_USERINFO_PART,
+			   context->pool, &url, &error) != 0) {
+		auth_request_log_error(context->request, "policy",
+			"Could not parse url %s: %s", context->url, error);
+		auth_policy_finish(context);
+		return;
+	}
+	context->http_request = http_client_request_url(http_client,
+		"POST", url, auth_policy_process_response, (void*)context);
+	http_client_request_set_destroy_callback(context->http_request, auth_policy_finish, context);
+	http_client_request_add_header(context->http_request, "Content-Type", "application/json");
+	if (*context->set->policy_server_api_header != 0) {
+		const char *ptr;
+		if ((ptr = strstr(context->set->policy_server_api_header, ":")) != NULL) {
+			const char *header = t_strcut(context->set->policy_server_api_header, ':');
+			http_client_request_add_header(context->http_request, header, ptr + 1);
+		} else {
+			http_client_request_add_header(context->http_request,
+				"X-API-Key", context->set->policy_server_api_header);
+		}
+	}
+	if (url->user != NULL) {
+		/* allow empty password */
+		http_client_request_set_auth_simple(context->http_request, url->user,
+			(url->password != NULL ? url->password : ""));
+	}
+	struct istream *is = i_stream_create_from_buffer(context->json);
+	http_client_request_set_payload(context->http_request, is, FALSE);
+	i_stream_unref(&is);
+	http_client_request_submit(context->http_request);
+}
+
+static
+const char *auth_policy_escape_function(const char *string,
+	const struct auth_request *auth_request ATTR_UNUSED)
+{
+	string_t *tmp = t_str_new(64);
+	json_append_escaped(tmp, string);
+	return str_c(tmp);
+}
+
+static
+const struct var_expand_table *policy_get_var_expand_table(struct auth_request *auth_request,
+	const char *hashed_password)
+{
+	struct var_expand_table *table;
+	unsigned int count = 1;
+
+	table = auth_request_get_var_expand_table_full(auth_request, auth_policy_escape_function,
+						       &count);
+	table[0].key = '\0';
+	table[0].long_key = "hashed_password";
+	table[0].value = hashed_password;
+	if (table[0].value != NULL)
+		table[0].value = auth_policy_escape_function(table[0].value, auth_request);
+
+	return table;
+}
+
+static
+void auth_policy_create_json(struct policy_lookup_ctx *context,
+	const char *password, bool include_success)
+{
+	const struct var_expand_table *var_table;
+	context->json = str_new(context->pool, 64);
+	unsigned char *ptr;
+	const struct hash_method *digest = hash_method_lookup(context->set->policy_hash_mech);
+
+	i_assert(digest != NULL);
+
+	void *ctx = t_malloc(digest->context_size);
+	string_t *buffer = t_str_new(64);
+
+	digest->init(ctx);
+	digest->loop(ctx,
+		context->set->policy_hash_nonce,
+		strlen(context->set->policy_hash_nonce));
+	/* use +1 to make sure \0 gets included */
+	digest->loop(ctx, context->request->user, strlen(context->request->user) + 1);
+	if (password != NULL)
+		digest->loop(ctx, password, strlen(password));
+	ptr = (unsigned char*)str_c_modifiable(buffer);
+	digest->result(ctx, ptr);
+	str_truncate(buffer, digest->digest_size);
+	if (context->set->policy_hash_truncate > 0) {
+		/* truncate it to closest byte boundary */
+		int bytes = ((context->set->policy_hash_truncate + 7) & -8)/8;
+		/* remainding bits */
+		int bits = bytes*8 - context->set->policy_hash_truncate;
+		str_truncate(buffer, bytes);
+		ptr = buffer_get_modifiable_data(buffer, NULL);
+		/* right shift everything and left-pad with 0 */
+		if (bits > 0) {
+			for(size_t i=bytes-1;i>0;i--)
+				ptr[i] = (ptr[i]>>(8-bits)) + ((ptr[i-1]&(0xff>>(8-bits)))<<bits);
+			ptr[0] = ptr[0]>>(8-bits);
+		}
+	}
+	const char *hashed_password = binary_to_hex(str_data(buffer), str_len(buffer));
+	str_append_c(context->json, '{');
+	var_table = policy_get_var_expand_table(context->request, hashed_password);
+	auth_request_var_expand_with_table(context->json, auth_policy_json_template,
+					   context->request, var_table,
+					   auth_policy_escape_function);
+	if (include_success) {
+		str_append(context->json, ",\"success\":");
+		if (!context->request->failed && context->request->successful &&
+		    !context->request->internal_failure)
+			str_append(context->json, "true");
+		else
+			str_append(context->json, "false");
+		str_append(context->json, ",\"policy_reject\":");
+		str_append(context->json, context->request->policy_refusal ? "true" : "false");
+	}
+	str_append_c(context->json, '}');
+	auth_request_log_debug(context->request, "policy",
+		"Policy server request JSON: %s", str_c(context->json));
+}
+
+static
+void auth_policy_url(struct policy_lookup_ctx *context, const char *command)
+{
+	size_t len = strlen(context->set->policy_server_url);
+	if (context->set->policy_server_url[len-1] == '&')
+		context->url = p_strdup_printf(context->pool, "%scommand=%s",
+			context->set->policy_server_url, command);
+	else
+		context->url = p_strdup_printf(context->pool, "%s?command=%s",
+			context->set->policy_server_url, command);
+}
+
+void auth_policy_check(struct auth_request *request, const char *password,
+	auth_policy_callback_t cb, void *context)
+{
+	if (*(request->set->policy_server_url) == '\0') {
+		cb(0, context);
+		return;
+	}
+	struct policy_lookup_ctx *ctx = p_new(request->pool, struct policy_lookup_ctx, 1);
+	ctx->pool = request->pool;
+	ctx->request = request;
+	ctx->expect_result = TRUE;
+	ctx->callback = cb;
+	ctx->callback_context = context;
+	ctx->set = request->set;
+
+	auth_policy_url(ctx, "allow");
+	ctx->result = (ctx->set->policy_reject_on_fail ? -1 : 0);
+	auth_request_log_debug(request, "policy", "Policy request %s", ctx->url);
+	T_BEGIN {
+		auth_policy_create_json(ctx, password, FALSE);
+	} T_END;
+	auth_policy_send_request(ctx);
+}
+
+void auth_policy_report(struct auth_request *request)
+{
+	if (*(request->set->policy_server_url) == '\0')
+		return;
+	pool_t pool = pool_alloconly_create("auth policy", 128);
+	struct policy_lookup_ctx *ctx = p_new(pool, struct policy_lookup_ctx, 1);
+	ctx->pool = pool;
+	ctx->request = request;
+	ctx->expect_result = FALSE;
+	ctx->set = request->set;
+	auth_policy_url(ctx, "report");
+	auth_request_log_debug(request, "policy", "Policy request %s", ctx->url);
+	T_BEGIN {
+		auth_policy_create_json(ctx, request->mech_password, TRUE);
+	} T_END;
+	auth_policy_send_request(ctx);
+}
diff --git a/src/auth/policy.h b/src/auth/policy.h
new file mode 100644
index 00000000000..590d0f834a2
--- /dev/null
+++ b/src/auth/policy.h
@@ -0,0 +1,8 @@
+#pragma once
+
+typedef void (*auth_policy_callback_t)(int, void *);
+
+void auth_policy_check(struct auth_request *request, const char *password, auth_policy_callback_t cb, void *context);
+void auth_policy_report(struct auth_request *request);
+void auth_policy_init(void);
+void auth_policy_deinit(void);
diff --git a/src/config/settings-get.pl b/src/config/settings-get.pl
index 7c6edfeea0b..6bfb1598d1e 100755
--- a/src/config/settings-get.pl
+++ b/src/config/settings-get.pl
@@ -10,6 +10,7 @@
 print '#include "hash-format.h"'."\n";
 print '#include "net.h"'."\n";
 print '#include "unichar.h"'."\n";
+print '#include "hash-method.h"'."\n";
 print '#include "settings-parser.h"'."\n";
 print '#include "all-settings.h"'."\n";
 print '#include <stddef.h>'."\n";

From 084bdd0ffeb5adb27d8f556389daa11eab26199d Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Fri, 3 Jun 2016 21:35:48 +0300
Subject: [PATCH 392/450] auth-policy: Hook auth policy to auth code

---
 src/auth/auth-request-handler.c    |   7 +-
 src/auth/auth-request-var-expand.c |   2 +-
 src/auth/auth-request-var-expand.h |   1 -
 src/auth/auth-request.c            | 144 ++++++++++++++++++++++++++++-
 src/auth/auth-request.h            |   1 +
 src/auth/main.c                    |   3 +
 6 files changed, 151 insertions(+), 7 deletions(-)

diff --git a/src/auth/auth-request-handler.c b/src/auth/auth-request-handler.c
index b42f490ed0b..66b3d4193eb 100644
--- a/src/auth/auth-request-handler.c
+++ b/src/auth/auth-request-handler.c
@@ -16,7 +16,7 @@
 #include "auth-token.h"
 #include "auth-master-connection.h"
 #include "auth-request-handler.h"
-
+#include "policy.h"
 
 #define AUTH_FAILURE_DELAY_CHECK_MSECS 500
 
@@ -215,6 +215,8 @@ auth_request_handle_failure(struct auth_request *request, const char *reply)
 	auth_request_ref(request);
 	auth_request_handler_remove(handler, request);
 
+	auth_policy_report(request);
+
 	if (auth_fields_exists(request->extra_fields, "nodelay")) {
 		/* passdb specifically requested not to delay the reply. */
 		handler->callback(reply, handler->conn);
@@ -267,6 +269,9 @@ auth_request_handler_reply_success_finish(struct auth_request *request)
 		   process to pick it up. delete it */
 		auth_request_handler_remove(handler, request);
 	}
+
+	auth_policy_report(request);
+
 	handler->callback(str_c(str), handler->conn);
 }
 
diff --git a/src/auth/auth-request-var-expand.c b/src/auth/auth-request-var-expand.c
index 7c6ee1c3766..f1f041e473b 100644
--- a/src/auth/auth-request-var-expand.c
+++ b/src/auth/auth-request-var-expand.c
@@ -157,6 +157,7 @@ auth_request_get_var_expand_table_full(const struct auth_request *auth_request,
 	tab[29].value = strchr(orig_user, '@');
 	if (tab[29].value != NULL)
 		tab[29].value = escape_func(tab[29].value+1, auth_request);
+
 	if (auth_request->master_user != NULL)
 		auth_user = auth_request->master_user;
 	else
@@ -166,7 +167,6 @@ auth_request_get_var_expand_table_full(const struct auth_request *auth_request,
 	tab[32].value = strchr(auth_user, '@');
 	if (tab[32].value != NULL)
 		tab[32].value = escape_func(tab[32].value+1, auth_request);
-
 	return ret_tab;
 }
 
diff --git a/src/auth/auth-request-var-expand.h b/src/auth/auth-request-var-expand.h
index 2b0c2aa4d86..1362ec3f861 100644
--- a/src/auth/auth-request-var-expand.h
+++ b/src/auth/auth-request-var-expand.h
@@ -9,7 +9,6 @@ auth_request_escape_func_t(const char *string,
 #define AUTH_REQUEST_VAR_TAB_USERNAME_IDX 1
 #define AUTH_REQUEST_VAR_TAB_DOMAIN_IDX 2
 #define AUTH_REQUEST_VAR_TAB_COUNT 33
-
 extern const struct var_expand_table
 auth_request_var_expand_static_tab[AUTH_REQUEST_VAR_TAB_COUNT+1];
 
diff --git a/src/auth/auth-request.c b/src/auth/auth-request.c
index beb707e31ae..69157732432 100644
--- a/src/auth/auth-request.c
+++ b/src/auth/auth-request.c
@@ -18,6 +18,7 @@
 #include "auth-request-stats.h"
 #include "auth-client-connection.h"
 #include "auth-master-connection.h"
+#include "policy.h"
 #include "passdb.h"
 #include "passdb-blocking.h"
 #include "passdb-cache.h"
@@ -40,6 +41,22 @@ struct auth_request_proxy_dns_lookup_ctx {
 	struct dns_lookup *dns_lookup;
 };
 
+struct auth_policy_check_ctx {
+	enum {
+		AUTH_POLICY_CHECK_TYPE_PLAIN,
+		AUTH_POLICY_CHECK_TYPE_LOOKUP,
+		AUTH_POLICY_CHECK_TYPE_SUCCESS,
+	} type;
+	struct auth_request *request;
+	const char *scheme;
+	char *password;
+
+	buffer_t *success_data;
+
+	verify_plain_callback_t *callback_plain;
+	lookup_credentials_callback_t *callback_lookup;
+};
+
 const char auth_default_subsystems[2];
 
 unsigned int auth_request_state_count[AUTH_REQUEST_STATE_MAX];
@@ -49,6 +66,17 @@ static void get_log_prefix(string_t *str, struct auth_request *auth_request,
 static void
 auth_request_userdb_import(struct auth_request *request, const char *args);
 
+static
+void auth_request_verify_plain_continue(struct auth_request *request,
+					char *password,
+					verify_plain_callback_t *callback);
+static
+void auth_request_lookup_credentials_policy_continue(struct auth_request *request,
+						     const char *scheme,
+						     lookup_credentials_callback_t *callback);
+static
+void auth_request_policy_check_callback(int result, void *context);
+
 struct auth_request *
 auth_request_new(const struct mech_module *mech)
 {
@@ -98,6 +126,8 @@ void auth_request_set_state(struct auth_request *request,
 	if (request->state == state)
 		return;
 
+	i_assert(request->to_penalty == NULL);
+
 	i_assert(auth_request_state_count[request->state] > 0);
 	auth_request_state_count[request->state]--;
 	auth_request_state_count[state]++;
@@ -127,8 +157,23 @@ struct auth *auth_request_get_auth(struct auth_request *request)
 void auth_request_success(struct auth_request *request,
 			  const void *data, size_t data_size)
 {
-	struct auth_stats *stats;
+	i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE);
 
+	/* perform second policy lookup here */
+
+	struct auth_policy_check_ctx *ctx = p_new(request->pool, struct auth_policy_check_ctx, 1);
+	ctx->request = request;
+	ctx->success_data = buffer_create_dynamic(request->pool, data_size);
+	buffer_append(ctx->success_data, data, data_size);
+        ctx->type = AUTH_POLICY_CHECK_TYPE_SUCCESS;
+	auth_policy_check(request, ctx->password, auth_request_policy_check_callback, ctx);
+}
+
+static
+void auth_request_success_continue(struct auth_request *request,
+				   const void *data, size_t data_size)
+{
+	struct auth_stats *stats;
 	i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE);
 
 	if (request->failed || !request->passdb_success) {
@@ -710,7 +755,7 @@ auth_request_verify_plain_callback_finish(enum passdb_result result,
 	} else {
 		auth_request_ref(request);
 		request->passdb_result = result;
-		request->private_callback.verify_plain(result, request);
+		request->private_callback.verify_plain(request->passdb_result, request);
 		auth_request_unref(&request);
 	}
 }
@@ -775,10 +820,78 @@ static bool auth_request_is_disabled_master_user(struct auth_request *request)
 	return TRUE;
 }
 
+static
+void auth_request_policy_penalty_finish(void *context)
+{
+	struct auth_policy_check_ctx *ctx = context;
+
+	if (ctx->request->to_penalty != NULL)
+		timeout_remove(&ctx->request->to_penalty);
+
+	i_assert(ctx->request->state == AUTH_REQUEST_STATE_MECH_CONTINUE);
+
+	switch(ctx->type) {
+	case AUTH_POLICY_CHECK_TYPE_PLAIN:
+		auth_request_verify_plain_continue(ctx->request, ctx->password, ctx->callback_plain);
+		return;
+	case AUTH_POLICY_CHECK_TYPE_LOOKUP:
+		auth_request_lookup_credentials_policy_continue(ctx->request, ctx->scheme, ctx->callback_lookup);
+		return;
+	case AUTH_POLICY_CHECK_TYPE_SUCCESS:
+		auth_request_success_continue(ctx->request, ctx->success_data->data, ctx->success_data->used);
+		return;
+	default:
+		i_unreached();
+	}
+}
+
+static
+void auth_request_policy_check_callback(int result, void *context)
+{
+	struct auth_policy_check_ctx *ctx = context;
+
+	ctx->request->policy_processed = TRUE;
+
+	if (result == -1) {
+		/* fail it right here and now */
+		auth_request_fail(ctx->request);
+	} else if (ctx->type != AUTH_POLICY_CHECK_TYPE_SUCCESS && result > 0 && !ctx->request->no_penalty) {
+		ctx->request->to_penalty = timeout_add(result * 1000,
+				auth_request_policy_penalty_finish,
+				context);
+	} else {
+		auth_request_policy_penalty_finish(context);
+	}
+}
+
 void auth_request_verify_plain(struct auth_request *request,
 			       const char *password,
 			       verify_plain_callback_t *callback)
 {
+	struct auth_policy_check_ctx *ctx;
+
+	i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE);
+
+	ctx = p_new(request->pool, struct auth_policy_check_ctx, 1);
+	ctx->request = request;
+	if (request->mech_password == NULL)
+		ctx->password = p_strdup(request->pool, password);
+	else
+		ctx->password = request->mech_password;
+	ctx->callback_plain = callback;
+	ctx->type = AUTH_POLICY_CHECK_TYPE_PLAIN;
+
+	if (request->policy_processed)
+		auth_request_verify_plain_continue(request, ctx->password, callback);
+	else
+		auth_policy_check(request, ctx->password, auth_request_policy_check_callback, ctx);
+}
+
+static
+void auth_request_verify_plain_continue(struct auth_request *request,
+					char *password,
+					verify_plain_callback_t *callback) {
+
 	struct auth_passdb *passdb;
 	enum passdb_result result;
 	const char *cache_key;
@@ -799,7 +912,8 @@ void auth_request_verify_plain(struct auth_request *request,
 
         passdb = request->passdb;
 	if (request->mech_password == NULL)
-		request->mech_password = p_strdup(request->pool, password);
+		/* this is allocated on start */
+		request->mech_password = password;
 	else
 		i_assert(request->mech_password == password);
 	request->private_callback.verify_plain = callback;
@@ -919,6 +1033,27 @@ void auth_request_lookup_credentials_callback(enum passdb_result result,
 void auth_request_lookup_credentials(struct auth_request *request,
 				     const char *scheme,
 				     lookup_credentials_callback_t *callback)
+{
+	struct auth_policy_check_ctx *ctx;
+
+	i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE);
+
+	ctx = p_new(request->pool, struct auth_policy_check_ctx, 1);
+	ctx->request = request;
+	if (request->credentials_scheme == NULL)
+		ctx->scheme = p_strdup(request->pool, scheme);
+	else
+		ctx->scheme = request->credentials_scheme;
+	ctx->callback_lookup = callback;
+	ctx->type = AUTH_POLICY_CHECK_TYPE_LOOKUP;
+
+	auth_policy_check(request, ctx->password, auth_request_policy_check_callback, ctx);
+}
+
+static
+void auth_request_lookup_credentials_policy_continue(struct auth_request *request,
+						     const char *scheme,
+						     lookup_credentials_callback_t *callback)
 {
 	struct auth_passdb *passdb;
 	const char *cache_key, *cache_cred, *cache_scheme;
@@ -932,7 +1067,8 @@ void auth_request_lookup_credentials(struct auth_request *request,
 	}
 	passdb = request->passdb;
 
-	request->credentials_scheme = p_strdup(request->pool, scheme);
+	if (request->credentials_scheme == NULL)
+		request->credentials_scheme = p_strdup(request->pool, scheme);
 	request->private_callback.lookup_credentials = callback;
 
 	cache_key = passdb_cache == NULL ? NULL : passdb->cache_key;
diff --git a/src/auth/auth-request.h b/src/auth/auth-request.h
index 989d2c79279..3a58b5f684c 100644
--- a/src/auth/auth-request.h
+++ b/src/auth/auth-request.h
@@ -145,6 +145,7 @@ struct auth_request {
 	unsigned int userdb_prefetch_set:1;
 	unsigned int stats_sent:1;
 	unsigned int policy_refusal:1;
+	unsigned int policy_processed:1;
 
 	/* ... mechanism specific data ... */
 };
diff --git a/src/auth/main.c b/src/auth/main.c
index 5a87c575ced..99b68cb84f4 100644
--- a/src/auth/main.c
+++ b/src/auth/main.c
@@ -30,6 +30,7 @@
 #include "auth-master-connection.h"
 #include "auth-client-connection.h"
 #include "auth-postfix-connection.h"
+#include "policy.h"
 
 #include <unistd.h>
 #include <sys/stat.h>
@@ -246,6 +247,7 @@ static void main_init(void)
 	auth_worker_server_init();
 	auths_init();
 	auth_request_handler_init();
+	auth_policy_init();
 
 	if (worker) {
 		/* workers have only a single connection from the master
@@ -265,6 +267,7 @@ static void main_deinit(void)
 		/* cancel all pending anvil penalty lookups */
 		auth_penalty_deinit(&auth_penalty);
 	}
+	auth_policy_deinit();
 	/* deinit auth workers, which aborts pending requests */
         auth_worker_server_deinit();
 	/* deinit passdbs and userdbs. it aborts any pending async requests. */

From 80b88a63fc301e9906f39b6cb26e2cd798fc686f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 28 Jun 2016 10:15:02 +0300
Subject: [PATCH 393/450] auth: Compiler warning fixes

---
 src/auth/policy.c | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/auth/policy.c b/src/auth/policy.c
index cbbd6e889e7..0deb9ee5815 100755
--- a/src/auth/policy.c
+++ b/src/auth/policy.c
@@ -35,7 +35,7 @@ struct policy_lookup_ctx {
 	struct auth_request *request;
 	struct http_client_request *http_request;
 	struct json_parser *parser;
-	struct auth_settings *set;
+	const struct auth_settings *set;
 	const char *url;
 	bool expect_result;
 	int result;
@@ -126,7 +126,7 @@ void auth_policy_open_and_close_to_key(const char *fromkey, const char *tokey, s
 	fptr = fromkey;
 	tptr = tokey;
 
-	while(fptr && tptr) {
+	while (fptr != NULL && tptr != NULL) {
 		fdash = strchr(fptr, '/');
 		tdash = strchr(tptr, '/');
 
@@ -313,7 +313,7 @@ void auth_policy_parse_response(struct policy_lookup_ctx *context)
 			(context->message!=NULL?context->message:""));
 	}
 
-	if (context->callback) {
+	if (context->callback != NULL) {
 		context->callback(context->result, context->callback_context);
 	}
 };
@@ -329,7 +329,7 @@ void auth_policy_process_response(const struct http_response *response,
 	if ((response->status / 10) != 20) {
 		auth_request_log_error(context->request, "policy",
 			"Policy server HTTP error: %d %s", response->status, response->reason);
-		if (context->callback)
+		if (context->callback != NULL)
 			context->callback(context->result, context->callback_context);
 		return;
 	}
@@ -338,7 +338,7 @@ void auth_policy_process_response(const struct http_response *response,
 		if (context->expect_result)
 			auth_request_log_error(context->request, "policy",
 				"Policy server result was empty");
-		if (context->callback)
+		if (context->callback != NULL)
 			context->callback(context->result, context->callback_context);
 		return;
 	}
@@ -351,7 +351,7 @@ void auth_policy_process_response(const struct http_response *response,
 	} else {
 		auth_request_log_debug(context->request, "policy",
 			"Policy response %d", context->result);
-		if (context->callback)
+		if (context->callback != NULL)
 			context->callback(context->result, context->callback_context);
 	}
 }

From 2c8e5ee65aa9dd2e167cd693273fafad3d11af99 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 28 Jun 2016 10:25:21 +0300
Subject: [PATCH 394/450] dict-ldap: Fixed linking with OSX

---
 src/plugins/dict-ldap/Makefile.am | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/plugins/dict-ldap/Makefile.am b/src/plugins/dict-ldap/Makefile.am
index 7caeba67c91..a03b9156de3 100644
--- a/src/plugins/dict-ldap/Makefile.am
+++ b/src/plugins/dict-ldap/Makefile.am
@@ -5,9 +5,9 @@ AM_CPPFLAGS = \
 	-I$(top_srcdir)/src/lib-settings
 
 LIBDICT_LDAP = libdict_ldap.la
-libdict_ldap_la_DEPENDENCIES = $(LIBDOVECOT_LDAP)
+libdict_ldap_la_DEPENDENCIES = $(LIBDOVECOT_LDAP) $(LIBDOVECOT_DEPS)
 libdict_ldap_la_LDFLAGS = -module -avoid-version
-libdict_ldap_la_LIBADD = $(LIBDOVECOT_LDAP)
+libdict_ldap_la_LIBADD = $(LIBDOVECOT_LDAP) $(LIBDOVECOT)
 
 module_dictdir = $(moduledir)/dict
 module_dict_LTLIBRARIES = \

From 5157c3c4429c0d4756971a1b29e5adbbf3a74290 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 28 Jun 2016 19:45:12 +0300
Subject: [PATCH 395/450] lazy-expunge: Fixed crash when copying from internal
 namespace

Most importantly fixes crash in LDA.
---
 src/plugins/lazy-expunge/lazy-expunge-plugin.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/plugins/lazy-expunge/lazy-expunge-plugin.c b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
index 16efb8ef209..1fdbe7fd1ff 100644
--- a/src/plugins/lazy-expunge/lazy-expunge-plugin.c
+++ b/src/plugins/lazy-expunge/lazy-expunge-plugin.c
@@ -348,7 +348,8 @@ static int lazy_expunge_copy(struct mail_save_context *ctx, struct mail *_mail)
 		LAZY_EXPUNGE_CONTEXT(ctx->transaction->box);
 	struct lazy_expunge_mail *mmail = LAZY_EXPUNGE_MAIL_CONTEXT(mail);
 
-	mmail->moving = ctx->moving;
+	if (mmail != NULL)
+		mmail->moving = ctx->moving;
 	return mbox->super.copy(ctx, _mail);
 }
 

From e1295b9399d9437d842a8b4832992745923c55c6 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 28 Jun 2016 22:01:29 +0300
Subject: [PATCH 396/450] lib: Added assert to iostream-temp

Input stream isn't expected to shrink here. Potentially this could be
changed to an error instead.
---
 src/lib/iostream-temp.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/lib/iostream-temp.c b/src/lib/iostream-temp.c
index 5fe4fc4b5a8..907837b102d 100644
--- a/src/lib/iostream-temp.c
+++ b/src/lib/iostream-temp.c
@@ -159,6 +159,7 @@ static int o_stream_temp_dup_istream(struct temp_ostream *outstream,
 			return o_stream_temp_dup_cancel(outstream);
 		return 0;
 	}
+	i_assert(instream->v_offset <= in_size);
 
 	if (outstream->dupstream == NULL) {
 		outstream->dupstream = instream;
@@ -185,8 +186,10 @@ static off_t o_stream_temp_send_istream(struct ostream_private *_outstream,
 
 	if ((outstream->flags & IOSTREAM_TEMP_FLAG_TRY_FD_DUP) != 0) {
 		orig_offset = outstream->dupstream_offset;
-		if ((ret = o_stream_temp_dup_istream(outstream, instream)) > 0)
+		if ((ret = o_stream_temp_dup_istream(outstream, instream)) > 0) {
+			i_assert(outstream->dupstream_offset >= orig_offset);
 			return outstream->dupstream_offset - orig_offset;
+		}
 		if (ret < 0)
 			return -1;
 		outstream->flags &= ~IOSTREAM_TEMP_FLAG_TRY_FD_DUP;

From be2be317de8059c135bea0ec698045f0f7475d6e Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 00:37:09 +0300
Subject: [PATCH 397/450] configure: Detect SSL_COMP_free_compression_methods()
 by linking

Fixes using libressl.
---
 configure.ac                                  | 3 +++
 src/lib-ssl-iostream/dovecot-openssl-common.c | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/configure.ac b/configure.ac
index 39b0c86abe5..e3283d0ee39 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1692,6 +1692,9 @@ if test $want_openssl != no && test $have_ssl = no; then
     AC_CHECK_LIB(ssl, SSL_get_servername, [
       AC_DEFINE(HAVE_SSL_GET_SERVERNAME,, [Build with TLS hostname support])
     ],, $SSL_LIBS)
+    AC_CHECK_LIB(ssl, SSL_COMP_free_compression_methods, [
+      AC_DEFINE(HAVE_SSL_COMP_FREE_COMPRESSION_METHODS,, [Build with SSL_COMP_free_compression_methods() support])
+    ],, $SSL_LIBS)
   fi
 fi
 AM_CONDITIONAL(BUILD_OPENSSL, test "$have_openssl" = "yes")
diff --git a/src/lib-ssl-iostream/dovecot-openssl-common.c b/src/lib-ssl-iostream/dovecot-openssl-common.c
index 2e1b8a03c33..8e54ec74451 100644
--- a/src/lib-ssl-iostream/dovecot-openssl-common.c
+++ b/src/lib-ssl-iostream/dovecot-openssl-common.c
@@ -71,7 +71,7 @@ bool dovecot_openssl_common_global_unref(void)
 #if OPENSSL_VERSION_NUMBER < 0x10001000L
 	OBJ_cleanup();
 #endif
-#if OPENSSL_VERSION_NUMBER >= 0x10002000L
+#ifdef HAVE_SSL_COMP_FREE_COMPRESSION_METHODS
 	SSL_COMP_free_compression_methods();
 #endif
 	ENGINE_cleanup();

From 223cfcf8c281c7971a434c74a40727be8634df7e Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Wed, 29 Jun 2016 13:52:09 +0300
Subject: [PATCH 398/450] auth-policy: Report success earlier

---
 src/auth/auth-request-handler.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/auth/auth-request-handler.c b/src/auth/auth-request-handler.c
index 66b3d4193eb..8b9fbf7dab5 100644
--- a/src/auth/auth-request-handler.c
+++ b/src/auth/auth-request-handler.c
@@ -262,6 +262,9 @@ auth_request_handler_reply_success_finish(struct auth_request *request)
 	str_printfa(str, "OK\t%u\tuser=", request->id);
 	str_append_tabescaped(str, request->user);
 	auth_str_append_extra_fields(request, str);
+
+	auth_policy_report(request);
+
 	if (handler->master_callback == NULL ||
 	    auth_fields_exists(request->extra_fields, "nologin") ||
 	    auth_fields_exists(request->extra_fields, "proxy")) {
@@ -270,8 +273,6 @@ auth_request_handler_reply_success_finish(struct auth_request *request)
 		auth_request_handler_remove(handler, request);
 	}
 
-	auth_policy_report(request);
-
 	handler->callback(str_c(str), handler->conn);
 }
 

From 3b28618dd9a617e1accf6523942d6c062a660785 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Wed, 29 Jun 2016 14:12:29 +0300
Subject: [PATCH 399/450] auth-policy: Do not allow/report when master query

This way auth policy isn't consulted when e.g. doveadm is used.
---
 src/auth/policy.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/auth/policy.c b/src/auth/policy.c
index 0deb9ee5815..d8e423ab708 100755
--- a/src/auth/policy.c
+++ b/src/auth/policy.c
@@ -495,7 +495,7 @@ void auth_policy_url(struct policy_lookup_ctx *context, const char *command)
 void auth_policy_check(struct auth_request *request, const char *password,
 	auth_policy_callback_t cb, void *context)
 {
-	if (*(request->set->policy_server_url) == '\0') {
+	if (request->master != NULL || *(request->set->policy_server_url) == '\0') {
 		cb(0, context);
 		return;
 	}
@@ -518,6 +518,9 @@ void auth_policy_check(struct auth_request *request, const char *password,
 
 void auth_policy_report(struct auth_request *request)
 {
+	if (request->master != NULL)
+		return;
+
 	if (*(request->set->policy_server_url) == '\0')
 		return;
 	pool_t pool = pool_alloconly_create("auth policy", 128);

From 5f8a6be68fafccd8aa305a9640d21d7e865787eb Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 14:16:58 +0300
Subject: [PATCH 400/450] lib-index: Fixed view syncing when changes couldn't
 be read from transaction logs

Fixes errors like:

Log synchronization error at seq=0,offset=0 for .../dovecot.index: Append with UID 5, but next_uid = 6
.../dovecot.index view syncing failed to apply changes
---
 src/lib-index/mail-index-view-sync.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-index/mail-index-view-sync.c b/src/lib-index/mail-index-view-sync.c
index d7c664ef3b1..2911d9d04b5 100644
--- a/src/lib-index/mail-index-view-sync.c
+++ b/src/lib-index/mail-index-view-sync.c
@@ -428,7 +428,7 @@ view_sync_get_log_lost_changes(struct mail_index_view_sync_ctx *ctx,
 
 	/* handle expunges and sync flags */
 	seqi = seqj = 1;
-	while (seqi < old_count && seqj < new_count) {
+	while (seqi <= old_count && seqj <= new_count) {
 		old_rec = MAIL_INDEX_REC_AT_SEQ(old_map, seqi);
 		new_rec = MAIL_INDEX_REC_AT_SEQ(new_map, seqj);
 		if (old_rec->uid == new_rec->uid) {

From 3dba61671ddc02668f71c9692eb39ea8f8a30db1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martti=20Rannanj=C3=A4rvi?= <martti.rannanjarvi@dovecot.fi>
Date: Tue, 28 Jun 2016 13:24:09 +0300
Subject: [PATCH 401/450] lib-dcrypt: correctly set version 2 on key info

Dovecot format version 2 keys were incorrectly reported as version 1
before.
---
 src/lib-dcrypt/dcrypt-openssl.c |  6 +++++-
 src/lib-dcrypt/test-crypto.c    | 30 ++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/src/lib-dcrypt/dcrypt-openssl.c b/src/lib-dcrypt/dcrypt-openssl.c
index 6447a46c439..f967695e643 100644
--- a/src/lib-dcrypt/dcrypt-openssl.c
+++ b/src/lib-dcrypt/dcrypt-openssl.c
@@ -1809,7 +1809,7 @@ bool dcrypt_openssl_key_string_get_info(const char *key_data, enum dcrypt_key_fo
 				return FALSE;
 			}
 		} else if (strcmp(fields[0], "2") == 0) {
-			version = DCRYPT_KEY_VERSION_1;
+			version = DCRYPT_KEY_VERSION_2;
 			if (nfields == 3) {
 				kind = DCRYPT_KEY_KIND_PUBLIC;
 			} else if (nfields == 5 && strcmp(fields[2],"0") == 0) {
@@ -1828,6 +1828,10 @@ bool dcrypt_openssl_key_string_get_info(const char *key_data, enum dcrypt_key_fo
 					*error_r = "Invalid dovecot v2 encoding";
 				return FALSE;
 			}
+		} else {
+			if (error_r != NULL)
+				*error_r = "Invalid dovecot key version";
+			return FALSE;
 		}
 
 		/* last field is always key hash */
diff --git a/src/lib-dcrypt/test-crypto.c b/src/lib-dcrypt/test-crypto.c
index a6d96b10b79..501bae414b1 100644
--- a/src/lib-dcrypt/test-crypto.c
+++ b/src/lib-dcrypt/test-crypto.c
@@ -440,6 +440,35 @@ void test_load_v2_public_key(void)
 	test_end();
 }
 
+static
+void test_get_info_v2_key(void) {
+	test_begin("test_get_info_v2_key");
+
+	const char *key = "2\t305e301006072a8648ce3d020106052b81040026034a000203fcc90034fa03d6fb79a0fc8b3b43c3398f68e76029307360cdcb9e27bb7e84b3c19dfb7244763bc4d442d216f09b7b7945ed9d182f3156550e9ee30b237a0217dbf79d28975f31\t86706b69d1f640011a65d26a42f2ba20a619173644e1cc7475eb1d90966e84dc";
+	enum dcrypt_key_format format;
+	enum dcrypt_key_version version = DCRYPT_KEY_VERSION_NA;
+	enum dcrypt_key_kind kind;
+	enum dcrypt_key_encryption_type encryption_type;
+	const char *encryption_key_hash = NULL;
+	const char *key_hash = NULL;
+	const char *error = NULL;
+
+	test_assert(dcrypt_key_string_get_info(key, &format, &version,
+			&kind, &encryption_type, &encryption_key_hash,
+			&key_hash, &error));
+	test_assert(error == NULL);
+	test_assert(format == DCRYPT_FORMAT_DOVECOT);
+	test_assert(version == DCRYPT_KEY_VERSION_2);
+
+	test_assert(kind == DCRYPT_KEY_KIND_PUBLIC);
+	test_assert(encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_NONE);
+	test_assert(encryption_key_hash == NULL);
+	test_assert(key_hash != NULL && strcmp(key_hash,
+		"86706b69d1f640011a65d26a42f2ba20a619173644e1cc7475eb1d90966e84dc") == 0);
+
+	test_end();
+}
+
 static
 void test_gen_and_get_info_rsa_pem(void)
 {
@@ -517,6 +546,7 @@ int main(void) {
 		test_load_v1_public_key,
 		test_load_v2_key,
 		test_load_v2_public_key,
+		test_get_info_v2_key,
 		test_gen_and_get_info_rsa_pem,
 		NULL
 	};

From e041c2d02472fe27a9235c59dad85253959f5f01 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 14:30:56 +0300
Subject: [PATCH 402/450] auth: Rename policy.[ch] to auth-policy.[ch]

---
 src/auth/{policy.c => auth-policy.c} | 0
 src/auth/{policy.h => auth-policy.h} | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename src/auth/{policy.c => auth-policy.c} (100%)
 rename src/auth/{policy.h => auth-policy.h} (100%)

diff --git a/src/auth/policy.c b/src/auth/auth-policy.c
similarity index 100%
rename from src/auth/policy.c
rename to src/auth/auth-policy.c
diff --git a/src/auth/policy.h b/src/auth/auth-policy.h
similarity index 100%
rename from src/auth/policy.h
rename to src/auth/auth-policy.h

From 8ce4a3a435aa61122c9ef2e0ce81a7ecc2b8e9fc Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 14:31:28 +0300
Subject: [PATCH 403/450] auth: Avoid nonstandard #pragma once

---
 src/auth/auth-policy.h | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/auth/auth-policy.h b/src/auth/auth-policy.h
index 590d0f834a2..1c81945d709 100644
--- a/src/auth/auth-policy.h
+++ b/src/auth/auth-policy.h
@@ -1,4 +1,5 @@
-#pragma once
+#ifndef AUTH_POLICY_H
+#define AUTH_POLICY_H
 
 typedef void (*auth_policy_callback_t)(int, void *);
 
@@ -6,3 +7,5 @@ void auth_policy_check(struct auth_request *request, const char *password, auth_
 void auth_policy_report(struct auth_request *request);
 void auth_policy_init(void);
 void auth_policy_deinit(void);
+
+#endif

From 3031ddfbdb5877d4fe963c73edc5f1bfee1e275b Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 14:43:31 +0300
Subject: [PATCH 404/450] auth: Finish policy.[ch] renaming..

---
 src/auth/Makefile.am            | 4 ++--
 src/auth/auth-policy.c          | 2 +-
 src/auth/auth-request-handler.c | 2 +-
 src/auth/auth-request.c         | 2 +-
 src/auth/main.c                 | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/auth/Makefile.am b/src/auth/Makefile.am
index 19c974f8a59..5c1987d6a3f 100644
--- a/src/auth/Makefile.am
+++ b/src/auth/Makefile.am
@@ -68,6 +68,7 @@ auth_SOURCES = \
 	auth-cache.c \
 	auth-client-connection.c \
 	auth-master-connection.c \
+	auth-policy.c \
 	auth-postfix-connection.c \
 	mech-otp-skey-common.c \
 	mech-plain-common.c \
@@ -118,7 +119,6 @@ auth_SOURCES = \
 	passdb-sql.c \
 	passdb-static.c \
 	passdb-template.c \
-	policy.c \
 	userdb.c \
 	userdb-blocking.c \
 	userdb-checkpassword.c \
@@ -143,6 +143,7 @@ headers = \
 	mech-otp-skey-common.h \
 	mech-plain-common.h \
 	auth-penalty.h \
+	auth-policy.h \
 	auth-request.h \
 	auth-request-handler.h \
 	auth-request-stats.h \
@@ -165,7 +166,6 @@ headers = \
 	passdb-cache.h \
 	passdb-template.h \
 	password-scheme.h \
-	policy.h \
 	userdb.h \
 	userdb-blocking.h \
 	userdb-template.h \
diff --git a/src/auth/auth-policy.c b/src/auth/auth-policy.c
index d8e423ab708..e815de1b028 100755
--- a/src/auth/auth-policy.c
+++ b/src/auth/auth-policy.c
@@ -12,7 +12,7 @@
 #include "auth-request.h"
 #include "auth-penalty.h"
 #include "auth-settings.h"
-#include "policy.h"
+#include "auth-policy.h"
 
 #define AUTH_POLICY_DNS_SOCKET_PATH "dns-client"
 
diff --git a/src/auth/auth-request-handler.c b/src/auth/auth-request-handler.c
index 8b9fbf7dab5..17d52c655e9 100644
--- a/src/auth/auth-request-handler.c
+++ b/src/auth/auth-request-handler.c
@@ -16,7 +16,7 @@
 #include "auth-token.h"
 #include "auth-master-connection.h"
 #include "auth-request-handler.h"
-#include "policy.h"
+#include "auth-policy.h"
 
 #define AUTH_FAILURE_DELAY_CHECK_MSECS 500
 
diff --git a/src/auth/auth-request.c b/src/auth/auth-request.c
index 69157732432..6c2ff9030ec 100644
--- a/src/auth/auth-request.c
+++ b/src/auth/auth-request.c
@@ -18,7 +18,7 @@
 #include "auth-request-stats.h"
 #include "auth-client-connection.h"
 #include "auth-master-connection.h"
-#include "policy.h"
+#include "auth-policy.h"
 #include "passdb.h"
 #include "passdb-blocking.h"
 #include "passdb-cache.h"
diff --git a/src/auth/main.c b/src/auth/main.c
index 99b68cb84f4..acfc535a8b2 100644
--- a/src/auth/main.c
+++ b/src/auth/main.c
@@ -30,7 +30,7 @@
 #include "auth-master-connection.h"
 #include "auth-client-connection.h"
 #include "auth-postfix-connection.h"
-#include "policy.h"
+#include "auth-policy.h"
 
 #include <unistd.h>
 #include <sys/stat.h>

From 5659f5edb63e00efb728014c94afbd09732f49dc Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Wed, 29 Jun 2016 14:55:12 +0300
Subject: [PATCH 405/450] auth-policy: Do not do policy checks every time

---
 src/auth/auth-request.c | 43 ++++++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 20 deletions(-)

diff --git a/src/auth/auth-request.c b/src/auth/auth-request.c
index 6c2ff9030ec..cb9ba4607c4 100644
--- a/src/auth/auth-request.c
+++ b/src/auth/auth-request.c
@@ -872,19 +872,19 @@ void auth_request_verify_plain(struct auth_request *request,
 
 	i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE);
 
-	ctx = p_new(request->pool, struct auth_policy_check_ctx, 1);
-	ctx->request = request;
-	if (request->mech_password == NULL)
-		ctx->password = p_strdup(request->pool, password);
-	else
-		ctx->password = request->mech_password;
-	ctx->callback_plain = callback;
-	ctx->type = AUTH_POLICY_CHECK_TYPE_PLAIN;
-
 	if (request->policy_processed)
 		auth_request_verify_plain_continue(request, ctx->password, callback);
-	else
+	else {
+		ctx = p_new(request->pool, struct auth_policy_check_ctx, 1);
+		ctx->request = request;
+		if (request->mech_password == NULL)
+			ctx->password = p_strdup(request->pool, password);
+		else
+			ctx->password = request->mech_password;
+		ctx->callback_plain = callback;
+		ctx->type = AUTH_POLICY_CHECK_TYPE_PLAIN;
 		auth_policy_check(request, ctx->password, auth_request_policy_check_callback, ctx);
+	}
 }
 
 static
@@ -1038,16 +1038,19 @@ void auth_request_lookup_credentials(struct auth_request *request,
 
 	i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE);
 
-	ctx = p_new(request->pool, struct auth_policy_check_ctx, 1);
-	ctx->request = request;
-	if (request->credentials_scheme == NULL)
-		ctx->scheme = p_strdup(request->pool, scheme);
-	else
-		ctx->scheme = request->credentials_scheme;
-	ctx->callback_lookup = callback;
-	ctx->type = AUTH_POLICY_CHECK_TYPE_LOOKUP;
-
-	auth_policy_check(request, ctx->password, auth_request_policy_check_callback, ctx);
+	if (request->policy_processed)
+		auth_request_lookup_credentials_policy_continue(request, scheme, callback);
+	else {
+		ctx = p_new(request->pool, struct auth_policy_check_ctx, 1);
+		ctx->request = request;
+		if (request->credentials_scheme == NULL)
+			ctx->scheme = p_strdup(request->pool, scheme);
+		else
+			ctx->scheme = request->credentials_scheme;
+		ctx->callback_lookup = callback;
+		ctx->type = AUTH_POLICY_CHECK_TYPE_LOOKUP;
+		auth_policy_check(request, ctx->password, auth_request_policy_check_callback, ctx);
+	}
 }
 
 static

From dbad2ad248cfa6fffb7c03bb84cdaafc910105b6 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Mon, 20 Jun 2016 14:43:04 +0300
Subject: [PATCH 406/450] lib-storage: Fixed MAILBOX_METADATA_FIRST_SAVE_DATE
 with mailbox_list_index=no

This also meant that autoexpunging wasn't working then.
---
 src/lib-storage/index/index-status.c | 43 ++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/src/lib-storage/index/index-status.c b/src/lib-storage/index/index-status.c
index 0c1c384078b..574339060ef 100644
--- a/src/lib-storage/index/index-status.c
+++ b/src/lib-storage/index/index-status.c
@@ -264,6 +264,45 @@ static void get_metadata_precache_fields(struct mailbox *box,
 	metadata_r->precache_fields = cache;
 }
 
+static int
+index_mailbox_get_first_save_date(struct mailbox *box,
+				  struct mailbox_metadata *metadata_r)
+{
+	const struct mail_index_header *hdr;
+	struct mailbox_transaction_context *t;
+	struct mail *mail;
+	uint32_t seq;
+	int ret = -1;
+
+	hdr = mail_index_get_header(box->view);
+	if (hdr->messages_count == 0) {
+		metadata_r->first_save_date = (time_t)-1;
+		return 0;
+	}
+
+	t = mailbox_transaction_begin(box, 0);
+	mail = mail_alloc(t, 0, NULL);
+	for (seq = 1; seq <= hdr->messages_count; seq++) {
+		mail_set_seq(mail, seq);
+		if (mail_get_save_date(mail, &metadata_r->first_save_date) == 0) {
+			ret = 0;
+			break;
+		}
+		if (mailbox_get_last_mail_error(box) != MAIL_ERROR_EXPUNGED) {
+			/* failed */
+			break;
+		}
+	}
+	mail_free(&mail);
+	(void)mailbox_transaction_commit(&t);
+	if (seq > hdr->messages_count) {
+		/* all messages were expunged after all */
+		metadata_r->first_save_date = (time_t)-1;
+		return 0;
+	}
+	return ret;
+}
+
 int index_mailbox_get_metadata(struct mailbox *box,
 			       enum mailbox_metadata_items items,
 			       struct mailbox_metadata *metadata_r)
@@ -296,6 +335,10 @@ int index_mailbox_get_metadata(struct mailbox *box,
 		if (index_mailbox_get_physical_size(box, metadata_r) < 0)
 			return -1;
 	}
+	if ((items & MAILBOX_METADATA_FIRST_SAVE_DATE) != 0) {
+		if (index_mailbox_get_first_save_date(box, metadata_r) < 0)
+			return -1;
+	}
 	if ((items & MAILBOX_METADATA_CACHE_FIELDS) != 0)
 		get_metadata_cache_fields(box, metadata_r);
 	if ((items & MAILBOX_METADATA_PRECACHE_FIELDS) != 0)

From 2722e8d90a70fc86739fc41a0a3084a909fe9f4d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 14:07:34 +0300
Subject: [PATCH 407/450] lmtp: If user has rached lmtp_user_concurrency_limit,
 fail at RCPT TO stage.

Otherwise LMTP client would have to send the entire email body before
getting the failure.
---
 src/lmtp/client.h   |  4 ----
 src/lmtp/commands.c | 54 ++++++++++++++++++++++-----------------------
 2 files changed, 26 insertions(+), 32 deletions(-)

diff --git a/src/lmtp/client.h b/src/lmtp/client.h
index c5b8b441353..9a2117df26a 100644
--- a/src/lmtp/client.h
+++ b/src/lmtp/client.h
@@ -16,7 +16,6 @@ struct mail_recipient {
 
 	struct anvil_query *anvil_query;
 	struct mail_storage_service_user *service_user;
-	unsigned int parallel_count;
 };
 
 struct client_state {
@@ -26,9 +25,6 @@ struct client_state {
 	ARRAY(struct mail_recipient *) rcpt_to;
 	unsigned int rcpt_idx;
 
-	unsigned int anvil_queries;
-	bool anvil_pending_data_write;
-
 	unsigned int data_end_idx;
 
 	/* Initially we start writing to mail_data. If it grows too large,
diff --git a/src/lmtp/commands.c b/src/lmtp/commands.c
index 5528de970a5..737740e9942 100644
--- a/src/lmtp/commands.c
+++ b/src/lmtp/commands.c
@@ -41,8 +41,6 @@
 
 #define LMTP_PROXY_DEFAULT_TIMEOUT_MSECS (1000*125)
 
-static void client_input_data_write(struct client *client);
-
 int cmd_lhlo(struct client *client, const char *args)
 {
 	struct rfc822_parser_context parser;
@@ -600,21 +598,30 @@ lmtp_rcpt_to_is_over_quota(struct client *client,
 static void rcpt_anvil_lookup_callback(const char *reply, void *context)
 {
 	struct mail_recipient *rcpt = context;
+	struct client *client = rcpt->client;
+	unsigned int parallel_count;
 
-	i_assert(rcpt->client->state.anvil_queries > 0);
+	i_assert(rcpt->anvil_query != NULL);
 
 	rcpt->anvil_query = NULL;
 	if (reply == NULL) {
 		/* lookup failed */
-	} else if (str_to_uint(reply, &rcpt->parallel_count) < 0) {
+	} else if (str_to_uint(reply, &parallel_count) < 0) {
 		i_error("Invalid reply from anvil: %s", reply);
 	}
-	if (--rcpt->client->state.anvil_queries == 0 &&
-	    rcpt->client->state.anvil_pending_data_write) {
-		/* DATA command was finished, but we were still waiting on
-		   anvil before handling any users */
-		client_input_data_write(rcpt->client);
+
+	if (parallel_count < client->lmtp_set->lmtp_user_concurrency_limit) {
+		client_send_line(client, "250 2.1.5 OK");
+		array_append(&client->state.rcpt_to, &rcpt, 1);
+	} else {
+		client_send_line(client, ERRSTR_TEMP_USERDB_FAIL_PREFIX
+				 "Too many concurrent deliveries for user",
+				 rcpt->address);
+		mail_storage_service_user_free(&rcpt->service_user);
 	}
+
+	client_io_reset(client);
+	client_input_handle(client);
 }
 
 int cmd_rcpt(struct client *client, const char *args)
@@ -718,18 +725,22 @@ int cmd_rcpt(struct client *client, const char *args)
 		mail_storage_service_user_free(&rcpt->service_user);
 		return 0;
 	}
-	array_append(&client->state.rcpt_to, &rcpt, 1);
-	client_send_line(client, "250 2.1.5 OK");
 
-	if (client->lmtp_set->lmtp_user_concurrency_limit > 0) {
+	if (client->lmtp_set->lmtp_user_concurrency_limit == 0) {
+		array_append(&client->state.rcpt_to, &rcpt, 1);
+		client_send_line(client, "250 2.1.5 OK");
+		return 0;
+	} else {
 		const char *query = t_strconcat("LOOKUP\t",
 			master_service_get_name(master_service),
 			"/", str_tabescape(username), NULL);
-		client->state.anvil_queries++;
+		io_remove(&client->io);
 		rcpt->anvil_query = anvil_client_query(anvil, query,
 					rcpt_anvil_lookup_callback, rcpt);
+		/* stop processing further commands while anvil query is
+		   pending */
+		return rcpt->anvil_query != NULL ? 0 : -1;
 	}
-	return 0;
 }
 
 int cmd_quit(struct client *client, const char *args ATTR_UNUSED)
@@ -788,19 +799,9 @@ client_deliver(struct client *client, const struct mail_recipient *rcpt,
 	enum mail_error mail_error;
 	int ret;
 
-	i_assert(client->state.anvil_queries == 0);
-
 	input = mail_storage_service_user_get_input(rcpt->service_user);
 	username = t_strdup(input->username);
 
-	if (client->lmtp_set->lmtp_user_concurrency_limit > 0 &&
-	    rcpt->parallel_count >= client->lmtp_set->lmtp_user_concurrency_limit) {
-		client_send_line(client, ERRSTR_TEMP_USERDB_FAIL_PREFIX
-				 "Too many concurrent deliveries for user",
-				 rcpt->address);
-		return -1;
-	}
-
 	mail_set = mail_storage_service_user_get_mail_set(rcpt->service_user);
 	set_parser = mail_storage_service_user_get_settings_parser(rcpt->service_user);
 	if (client->proxy_timeout_secs > 0 &&
@@ -1250,10 +1251,7 @@ static void client_input_data_handle(struct client *client)
 		return;
 	}
 
-	if (client->state.anvil_queries == 0)
-		client_input_data_write(client);
-	else
-		client->state.anvil_pending_data_write = TRUE;
+	client_input_data_write(client);
 }
 
 static void client_input_data(struct client *client)

From f36ac996f90947eba0adcf389d9fa688fe6aa74f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 14:09:01 +0300
Subject: [PATCH 408/450] lmtp: Increase user's concurrency limit already after
 RCPT TO.

This way it's not possible for a lot of mails to arrive to user concurrently
and bypass the lmtp_user_concurrency_limit.
---
 src/lmtp/client.c   | 14 ++++++++++++++
 src/lmtp/client.h   |  2 ++
 src/lmtp/commands.c | 18 ++++++++----------
 3 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/src/lmtp/client.c b/src/lmtp/client.c
index 8f9d704c998..5650b412123 100644
--- a/src/lmtp/client.c
+++ b/src/lmtp/client.c
@@ -343,6 +343,19 @@ void client_disconnect(struct client *client, const char *prefix,
 	client->disconnected = TRUE;
 }
 
+void client_rcpt_anvil_disconnect(const struct mail_recipient *rcpt)
+{
+	const struct mail_storage_service_input *input;
+
+	if (!rcpt->anvil_connect_sent)
+		return;
+
+	input = mail_storage_service_user_get_input(rcpt->service_user);
+	master_service_anvil_send(master_service, t_strconcat(
+		"DISCONNECT\t", my_pid, "\t", master_service_get_name(master_service),
+		"/", input->username, "\n", NULL));
+}
+
 void client_state_reset(struct client *client, const char *state_name)
 {
 	struct mail_recipient *const *rcptp;
@@ -354,6 +367,7 @@ void client_state_reset(struct client *client, const char *state_name)
 		array_foreach_modifiable(&client->state.rcpt_to, rcptp) {
 			if ((*rcptp)->anvil_query != NULL)
 				anvil_client_query_abort(anvil, &(*rcptp)->anvil_query);
+			client_rcpt_anvil_disconnect(*rcptp);
 			mail_storage_service_user_free(&(*rcptp)->service_user);
 		}
 	}
diff --git a/src/lmtp/client.h b/src/lmtp/client.h
index 9a2117df26a..fe91244a6e0 100644
--- a/src/lmtp/client.h
+++ b/src/lmtp/client.h
@@ -15,6 +15,7 @@ struct mail_recipient {
 	struct lmtp_recipient_params params;
 
 	struct anvil_query *anvil_query;
+	bool anvil_connect_sent;
 	struct mail_storage_service_user *service_user;
 };
 
@@ -88,6 +89,7 @@ void client_destroy(struct client *client, const char *prefix,
 void client_disconnect(struct client *client, const char *prefix,
 		       const char *reason);
 void client_io_reset(struct client *client);
+void client_rcpt_anvil_disconnect(const struct mail_recipient *rcpt);
 void client_state_reset(struct client *client, const char *state_name);
 void client_state_set(struct client *client, const char *name, const char *args);
 const char *client_remote_id(struct client *client);
diff --git a/src/lmtp/commands.c b/src/lmtp/commands.c
index 737740e9942..71ca799c41c 100644
--- a/src/lmtp/commands.c
+++ b/src/lmtp/commands.c
@@ -599,6 +599,7 @@ static void rcpt_anvil_lookup_callback(const char *reply, void *context)
 {
 	struct mail_recipient *rcpt = context;
 	struct client *client = rcpt->client;
+	const struct mail_storage_service_input *input;
 	unsigned int parallel_count;
 
 	i_assert(rcpt->anvil_query != NULL);
@@ -612,6 +613,12 @@ static void rcpt_anvil_lookup_callback(const char *reply, void *context)
 
 	if (parallel_count < client->lmtp_set->lmtp_user_concurrency_limit) {
 		client_send_line(client, "250 2.1.5 OK");
+
+		rcpt->anvil_connect_sent = TRUE;
+		input = mail_storage_service_user_get_input(rcpt->service_user);
+		master_service_anvil_send(master_service, t_strconcat(
+			"CONNECT\t", my_pid, "\t", master_service_get_name(master_service),
+			"/", input->username, "\n", NULL));
 		array_append(&client->state.rcpt_to, &rcpt, 1);
 	} else {
 		client_send_line(client, ERRSTR_TEMP_USERDB_FAIL_PREFIX
@@ -875,11 +882,6 @@ client_deliver(struct client *client, const struct mail_recipient *rcpt,
 	dctx.save_dest_mail = array_count(&client->state.rcpt_to) > 1 &&
 		client->state.first_saved_mail == NULL;
 
-	if (client->lmtp_set->lmtp_user_concurrency_limit > 0) {
-		master_service_anvil_send(master_service, t_strconcat(
-			"CONNECT\t", my_pid, "\t", master_service_get_name(master_service),
-			"/", username, "\n", NULL));
-	}
 	if (mail_deliver(&dctx, &storage) == 0) {
 		if (dctx.dest_mail != NULL) {
 			i_assert(client->state.first_saved_mail == NULL);
@@ -908,11 +910,7 @@ client_deliver(struct client *client, const struct mail_recipient *rcpt,
 				 rcpt->address);
 		ret = -1;
 	}
-	if (client->lmtp_set->lmtp_user_concurrency_limit > 0) {
-		master_service_anvil_send(master_service, t_strconcat(
-			"DISCONNECT\t", my_pid, "\t", master_service_get_name(master_service),
-			"/", username, "\n", NULL));
-	}
+	client_rcpt_anvil_disconnect(rcpt);
 	return ret;
 }
 

From df0e630a976afa31a0016032bfe52ffc77388711 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 28 Jun 2016 22:20:20 +0300
Subject: [PATCH 409/450] auth: Added ":remove" suffix for passdb/userdb extra
 fields to remove fields.

---
 src/auth/auth-request.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/src/auth/auth-request.c b/src/auth/auth-request.c
index cb9ba4607c4..49d7a2fb26f 100644
--- a/src/auth/auth-request.c
+++ b/src/auth/auth-request.c
@@ -1650,6 +1650,11 @@ void auth_request_set_field(struct auth_request *request,
 		name = t_strndup(name, name_len-10);
 		if (auth_fields_exists(request->extra_fields, name))
 			return;
+	} else if (name_len > 7 && strcmp(name+name_len-7, ":remove") == 0) {
+		/* remove this field entirely */
+		name = t_strndup(name, name_len-7);
+		auth_fields_remove(request->extra_fields, name);
+		return;
 	}
 
 	if (strcmp(name, "password") == 0) {
@@ -1814,6 +1819,11 @@ void auth_request_set_userdb_field(struct auth_request *request,
 		name = t_strndup(name, name_len-10);
 		if (auth_fields_exists(request->userdb_reply, name))
 			return;
+	} else if (name_len > 7 && strcmp(name+name_len-7, ":remove") == 0) {
+		/* remove this field entirely */
+		name = t_strndup(name, name_len-7);
+		auth_fields_remove(request->userdb_reply, name);
+		return;
 	}
 
 	if (strcmp(name, "uid") == 0) {

From 0f8880fddc324a50c8671c2ac767d43a1786117e Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 15:49:38 +0300
Subject: [PATCH 410/450] lib-fts: Added maxlen parameter to icu-normalizer.

This is needed because the normalization can increase the token's length.
---
 src/lib-fts/fts-filter-normalizer-icu.c | 16 +++++++-
 src/lib-fts/fts-filter.h                |  6 ++-
 src/lib-fts/test-fts-filter.c           | 52 ++++++++++++++++++++-----
 3 files changed, 62 insertions(+), 12 deletions(-)

diff --git a/src/lib-fts/fts-filter-normalizer-icu.c b/src/lib-fts/fts-filter-normalizer-icu.c
index 4a9cf9fbc86..58b2a9b475f 100644
--- a/src/lib-fts/fts-filter-normalizer-icu.c
+++ b/src/lib-fts/fts-filter-normalizer-icu.c
@@ -4,6 +4,7 @@
 #include "buffer.h"
 #include "str.h"
 #include "unichar.h" /* unicode replacement char */
+#include "fts-tokenizer-common.h"
 #include "fts-filter-private.h"
 #include "fts-language.h"
 
@@ -18,6 +19,7 @@ struct fts_filter_normalizer_icu {
 	UTransliterator *transliterator;
 	buffer_t *utf16_token, *trans_token;
 	string_t *utf8_token;
+	unsigned int maxlen;
 };
 
 static void fts_filter_normalizer_icu_destroy(struct fts_filter *filter)
@@ -38,7 +40,7 @@ fts_filter_normalizer_icu_create(const struct fts_language *lang ATTR_UNUSED,
 {
 	struct fts_filter_normalizer_icu *np;
 	pool_t pp;
-	unsigned int i;
+	unsigned int i, max_length = 250;
 	const char *id = "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC; [\\x20] Remove";
 
 	for (i = 0; settings[i] != NULL; i += 2) {
@@ -46,6 +48,12 @@ fts_filter_normalizer_icu_create(const struct fts_language *lang ATTR_UNUSED,
 
 		if (strcmp(key, "id") == 0) {
 			id = value;
+		} else if (strcmp(key, "maxlen") == 0) {
+			if (str_to_uint(value, &max_length) < 0 ||
+			    max_length == 0) {
+				*error_r = t_strdup_printf("Invalid icu maxlen setting: %s", value);
+				return -1;
+			}
 		} else {
 			*error_r = t_strdup_printf("Unknown setting: %s", key);
 			return -1;
@@ -61,6 +69,7 @@ fts_filter_normalizer_icu_create(const struct fts_language *lang ATTR_UNUSED,
 	np->utf16_token = buffer_create_dynamic(pp, 128);
 	np->trans_token = buffer_create_dynamic(pp, 128);
 	np->utf8_token = buffer_create_dynamic(pp, 128);
+	np->maxlen = max_length;
 	*filter_r = &np->filter;
 	return 0;
 }
@@ -92,6 +101,11 @@ fts_filter_normalizer_icu_filter(struct fts_filter *filter, const char **token,
 
 	fts_icu_utf16_to_utf8(np->utf8_token, np->trans_token->data,
 			      np->trans_token->used / sizeof(UChar));
+	if (str_len(np->utf8_token) > np->maxlen) {
+		size_t len = np->maxlen;
+		fts_tokenizer_delete_trailing_partial_char(np->utf8_token->data, &len);
+		str_truncate(np->utf8_token, len);
+	}
 	*token = str_c(np->utf8_token);
 	return 1;
 }
diff --git a/src/lib-fts/fts-filter.h b/src/lib-fts/fts-filter.h
index e37bdff6656..0c89f2566e6 100644
--- a/src/lib-fts/fts-filter.h
+++ b/src/lib-fts/fts-filter.h
@@ -24,8 +24,10 @@ extern const struct fts_filter *fts_filter_stemmer_snowball;
  Settings: "id", description of the normalizing/translitterating rules
  to use.  See
  http://userguide.icu-project.org/transforms/general#TOC-Transliterator-Identifiers
- for syntax.  Defaults to "Any-Lower; NFKD; [: Nonspacing Mark :]
- Remove; NFC"
+ for syntax. Defaults to "Any-Lower; NFKD; [: Nonspacing Mark :] Remove; NFC"
+
+ "maxlen", maximum length of tokens that ICU normalizer will output.
+  Defaults to 250.
  */
 extern const struct fts_filter *fts_filter_normalizer_icu;
 
diff --git a/src/lib-fts/test-fts-filter.c b/src/lib-fts/test-fts-filter.c
index a3844b477a3..6c5f1a7b2c8 100644
--- a/src/lib-fts/test-fts-filter.c
+++ b/src/lib-fts/test-fts-filter.c
@@ -555,20 +555,20 @@ static void test_fts_filter_normalizer_french(void)
 	FILE *input;
 	const char * const settings[] =
 		{"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove", NULL};
-	char buf[4096] = {0};
+	char buf[250] = {0};
 	const char *error = NULL;
 	const char *tokens;
 	unsigned char sha512_digest[SHA512_RESULTLEN];
 	struct sha512_ctx ctx;
 	const unsigned char correct_digest[] = {
-		0x78, 0x1e, 0xb9, 0x04, 0xa4, 0x92, 0xca, 0x88,
-		0x1e, 0xef, 0x7b, 0xc8, 0x3e, 0x4a, 0xa8, 0xdb,
-		0x9c, 0xd4, 0x42, 0x5c, 0x64, 0x81, 0x06, 0xd5,
-		0x72, 0x93, 0x38, 0x0c, 0x09, 0xce, 0xbe, 0xdf,
-		0x65, 0xff, 0x36, 0x35, 0x05, 0x77, 0xcc, 0xc6,
-		0xff, 0x44, 0x2c, 0x31, 0x10, 0x00, 0xf6, 0x8d,
-		0x15, 0x25, 0x1e, 0x54, 0x67, 0x2a, 0x5b, 0xc1,
-		0xdb, 0x84, 0xc5, 0x0d, 0x43, 0x7e, 0x8c, 0x70};
+		0x06, 0x80, 0xf1, 0x81, 0xf2, 0xed, 0xfb, 0x6d,
+		0xcd, 0x7d, 0xcb, 0xbd, 0xc4, 0x87, 0xc3, 0xf6,
+		0xb8, 0x6a, 0x01, 0x82, 0xdf, 0x0a, 0xb5, 0x92,
+		0x6b, 0x9b, 0x7b, 0x21, 0x5e, 0x62, 0x40, 0xbd,
+		0xbf, 0x15, 0xb9, 0x7b, 0x75, 0x9c, 0x4e, 0xc9,
+		0xe8, 0x48, 0xaa, 0x08, 0x63, 0xf2, 0xa0, 0x6c,
+		0x20, 0x4c, 0x01, 0xe3, 0xb3, 0x4f, 0x15, 0xc6,
+		0x8c, 0xd6, 0x7a, 0xb7, 0xc5, 0xc6, 0x85, 0x00};
 	const char *udhr_path;
 
 	test_begin("fts filter normalizer French UDHR");
@@ -666,6 +666,39 @@ static void test_fts_filter_normalizer_invalid_id(void)
 	test_end();
 }
 
+static void test_fts_filter_normalizer_oversized(void)
+{
+	struct fts_filter *norm = NULL;
+	const char *settings[] =
+		{"id", "Any-Lower; NFKD; [: Nonspacing Mark :] Remove", "maxlen", "250",
+		 NULL};
+	const char *error = NULL;
+	const char *token = "\xe4\x95\x91\x25\xe2\x94\xad\xe1\x90\xad\xee\x94\x81\xe2\x8e\x9e"
+						"\xe7\x9a\xb7\xea\xbf\x97\xe3\xb2\x8f\xe4\x9c\xbe\xee\xb4\x98\xe1"
+						"\x8d\x99\xe2\x91\x83\xe3\xb1\xb8\xef\xbf\xbd\xe8\xbb\x9c\xef\xbf"
+						"\xbd\xea\xbb\x98\xea\xb5\xac\xe4\x87\xae\xe4\x88\x93\xe9\x86\x8f"
+						"\xe9\x86\x83\xe6\x8f\x8d\xe7\xa3\x9d\xed\x89\x96\xe2\x89\x85\xe6"
+						"\x8c\x82\xec\x80\x98\xee\x91\x96\xe7\xa8\x8a\xec\xbc\x85\xeb\x9c"
+						"\xbd\xeb\x97\x95\xe3\xa4\x9d\xd7\xb1\xea\xa7\x94\xe0\xbb\xac\xee"
+						"\x95\x87\xd5\x9d\xe8\xba\x87\xee\x8b\xae\xe5\xb8\x80\xe9\x8d\x82"
+						"\xe7\xb6\x8c\xe7\x9b\xa0\xef\x82\x9f\xed\x96\xa4\xe3\x8d\xbc\xe1"
+						"\x81\xbd\xe9\x81\xb2\xea\xac\xac\xec\x9b\x98\xe7\x84\xb2\xee\xaf"
+						"\xbc\xeb\xa2\x9d\xe9\x86\xb3\xe0\xb0\x89\xeb\x80\xb6\xe3\x8c\x9d"
+						"\xe9\x8f\x9e\xe2\xae\x8a\xee\x9e\x9a\xef\xbf\xbd\xe7\xa3\x9b\xe4"
+						"\xa3\x8b\xe4\x82\xb9\xeb\x8e\x93\xec\xb5\x82\xe5\xa7\x81\xe2\x8c"
+						"\x97\xea\xbb\xb4\xe5\x85\xb7\xeb\x96\xbe\xe7\x97\x91\xea\xbb\x98"
+						"\xe6\xae\xb4\xe9\x8a\x85\xc4\xb9\xe4\x90\xb2\xe9\x96\xad\xef\x90"
+						"\x9c\xe5\xa6\xae\xe9\x93\x91\xe8\x87\xa1";
+
+	test_begin("fts filter normalizer over-sized token");
+	test_assert(fts_filter_create(fts_filter_normalizer_icu, NULL, NULL, settings, &norm, &error) == 0);
+	test_assert(error == NULL);
+	test_assert(fts_filter_filter(norm, &token, &error) >= 0);
+	test_assert(strlen(token) <= 250);
+	fts_filter_unref(&norm);
+	test_end();
+}
+
 #ifdef HAVE_FTS_STEMMER
 static void test_fts_filter_normalizer_stopwords_stemmer_eng(void)
 {
@@ -922,6 +955,7 @@ int main(void)
 		test_fts_filter_normalizer_empty,
 		test_fts_filter_normalizer_baddata,
 		test_fts_filter_normalizer_invalid_id,
+		test_fts_filter_normalizer_oversized,
 #ifdef HAVE_FTS_STEMMER
 		test_fts_filter_normalizer_stopwords_stemmer_eng,
 		test_fts_filter_stopwords_normalizer_stemmer_no,

From 7dbddb3b9238d8c9ff0964195c65b7018d80cd06 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 16:01:13 +0300
Subject: [PATCH 411/450] auth: Fixed plaintext authentication when auth policy
 was already processed.

Broken by e82511362.
---
 src/auth/auth-request.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/auth/auth-request.c b/src/auth/auth-request.c
index 49d7a2fb26f..fd983aad091 100644
--- a/src/auth/auth-request.c
+++ b/src/auth/auth-request.c
@@ -872,9 +872,10 @@ void auth_request_verify_plain(struct auth_request *request,
 
 	i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE);
 
-	if (request->policy_processed)
-		auth_request_verify_plain_continue(request, ctx->password, callback);
-	else {
+	if (request->policy_processed) {
+		auth_request_verify_plain_continue(request,
+			p_strdup(request->pool, password), callback);
+	} else {
 		ctx = p_new(request->pool, struct auth_policy_check_ctx, 1);
 		ctx->request = request;
 		if (request->mech_password == NULL)

From 7e75c14dd9c10123245af0fefda826763e3c5278 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 15:25:28 +0300
Subject: [PATCH 412/450] auth: Added delay_until passdb extra field.

---
 src/auth/auth-request.c | 39 +++++++++++++++++++++++++++++++++------
 src/auth/auth-request.h |  1 +
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/src/auth/auth-request.c b/src/auth/auth-request.c
index fd983aad091..4e9a428b922 100644
--- a/src/auth/auth-request.c
+++ b/src/auth/auth-request.c
@@ -33,6 +33,7 @@
 #define AUTH_DNS_SOCKET_PATH "dns-client"
 #define AUTH_DNS_DEFAULT_TIMEOUT_MSECS (1000*10)
 #define AUTH_DNS_WARN_MSECS 500
+#define AUTH_REQUEST_MAX_DELAY_SECS (60*5)
 #define CACHED_PASSWORD_SCHEME "SHA1"
 
 struct auth_request_proxy_dns_lookup_ctx {
@@ -170,23 +171,34 @@ void auth_request_success(struct auth_request *request,
 }
 
 static
-void auth_request_success_continue(struct auth_request *request,
-				   const void *data, size_t data_size)
+void auth_request_success_continue(struct auth_policy_check_ctx *ctx)
 {
+	struct auth_request *request = ctx->request;
 	struct auth_stats *stats;
 	i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE);
 
+	if (request->to_penalty != NULL)
+		timeout_remove(&request->to_penalty);
+
 	if (request->failed || !request->passdb_success) {
 		/* password was valid, but some other check failed. */
 		auth_request_fail(request);
 		return;
 	}
 
+	if (request->delay_until > ioloop_time) {
+		unsigned int delay_secs = request->delay_until - ioloop_time;
+		request->to_penalty = timeout_add(delay_secs * 1000,
+			auth_request_success_continue, ctx);
+		return;
+	}
+
 	request->successful = TRUE;
-	if (data_size > 0 && !request->final_resp_ok) {
+	if (ctx->success_data->used > 0 && !request->final_resp_ok) {
 		/* we'll need one more SASL round, since client doesn't support
 		   the final SASL response */
-		auth_request_handler_reply_continue(request, data, data_size);
+		auth_request_handler_reply_continue(request,
+			ctx->success_data->data, ctx->success_data->used);
 		return;
 	}
 
@@ -198,7 +210,7 @@ void auth_request_success_continue(struct auth_request *request,
 	auth_request_set_state(request, AUTH_REQUEST_STATE_FINISHED);
 	auth_request_refresh_last_access(request);
 	auth_request_handler_reply(request, AUTH_CLIENT_RESULT_SUCCESS,
-				   data, data_size);
+		ctx->success_data->data, ctx->success_data->used);
 }
 
 void auth_request_fail(struct auth_request *request)
@@ -838,7 +850,7 @@ void auth_request_policy_penalty_finish(void *context)
 		auth_request_lookup_credentials_policy_continue(ctx->request, ctx->scheme, ctx->callback_lookup);
 		return;
 	case AUTH_POLICY_CHECK_TYPE_SUCCESS:
-		auth_request_success_continue(ctx->request, ctx->success_data->data, ctx->success_data->used);
+		auth_request_success_continue(ctx);
 		return;
 	default:
 		i_unreached();
@@ -1677,6 +1689,21 @@ void auth_request_set_field(struct auth_request *request,
 		auth_request_validate_networks(request, name, value, &request->remote_ip);
 	} else if (strcmp(name, "fail") == 0) {
 		request->failed = TRUE;
+	} else if (strcmp(name, "delay_until") == 0) {
+		time_t timestamp;
+
+		if (str_to_time(value, &timestamp) < 0) {
+			auth_request_log_error(request, AUTH_SUBSYS_DB,
+				"Invalid delay_until timestamp '%s'", value);
+		} else if (timestamp <= ioloop_time) {
+			/* no more delays */
+		} else if (ioloop_time - timestamp > AUTH_REQUEST_MAX_DELAY_SECS) {
+			auth_request_log_error(request, AUTH_SUBSYS_DB,
+				"delay_until timestamp %s is too much in the future, failing", value);
+			request->failed = TRUE;
+		} else {
+			request->delay_until = timestamp;
+		}
 	} else if (strcmp(name, "allow_real_nets") == 0) {
 		auth_request_validate_networks(request, name, value, &request->real_remote_ip);
 	} else if (strncmp(name, "userdb_", 7) == 0) {
diff --git a/src/auth/auth-request.h b/src/auth/auth-request.h
index 3a58b5f684c..de525776d00 100644
--- a/src/auth/auth-request.h
+++ b/src/auth/auth-request.h
@@ -71,6 +71,7 @@ struct auth_request {
 	unsigned int client_pid;
 	unsigned int id;
 	time_t last_access;
+	time_t delay_until;
 	pid_t session_pid;
 
 	const char *service, *mech_name, *session_id;

From 20a3cd140ef8f1087b23ba254f9ba80b574e92c4 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 16:34:11 +0300
Subject: [PATCH 413/450] lmtp: If anvil lookup fails, ignore
 lmtp_user_concurrency_limit

Previously it was randomly allowing or disallowing the client.
---
 src/lmtp/commands.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lmtp/commands.c b/src/lmtp/commands.c
index 71ca799c41c..b65c1bf7869 100644
--- a/src/lmtp/commands.c
+++ b/src/lmtp/commands.c
@@ -600,7 +600,7 @@ static void rcpt_anvil_lookup_callback(const char *reply, void *context)
 	struct mail_recipient *rcpt = context;
 	struct client *client = rcpt->client;
 	const struct mail_storage_service_input *input;
-	unsigned int parallel_count;
+	unsigned int parallel_count = 0;
 
 	i_assert(rcpt->anvil_query != NULL);
 

From d53679051cc1e408559fff8d4b398e7ee023e441 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Sat, 4 Jun 2016 08:14:45 +0300
Subject: [PATCH 414/450] doveadm-server: Skip doveadm_print_init

Some commands need to do output that is not doable with
doveadm_print, so we need to have a flag to indicate this.
---
 src/doveadm/client-connection-http.c | 9 ++++++---
 src/doveadm/doveadm-cmd.h            | 3 ++-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/doveadm/client-connection-http.c b/src/doveadm/client-connection-http.c
index 52191ebcb12..076f6bee750 100644
--- a/src/doveadm/client-connection-http.c
+++ b/src/doveadm/client-connection-http.c
@@ -329,8 +329,11 @@ doveadm_http_server_command_execute(struct client_connection_http *conn)
 
 	// create iostream
 	doveadm_print_ostream = iostream_temp_create("/tmp/doveadm.", 0);
+	cctx.cmd = conn->cmd;
+
+	if ((cctx.cmd->flags & CMD_FLAG_NO_PRINT) == 0)
+		doveadm_print_init(DOVEADM_PRINT_TYPE_JSON);
 
-	doveadm_print_init(DOVEADM_PRINT_TYPE_JSON);
 	/* then call it */
 	doveadm_cmd_params_null_terminate_arrays(&conn->pargv);
 	cctx.argv = array_get(&conn->pargv, (unsigned int*)&cctx.argc);
@@ -338,7 +341,6 @@ doveadm_http_server_command_execute(struct client_connection_http *conn)
 	lib_signals_reset_ioloop();
 	doveadm_exit_code = 0;
 
-	cctx.cmd = conn->cmd;
 	cctx.cli = FALSE;
 	cctx.local_ip = conn->client.local_ip;
 	cctx.local_port = conn->client.local_port;
@@ -357,7 +359,8 @@ doveadm_http_server_command_execute(struct client_connection_http *conn)
 	io_loop_set_current(ioloop);
 	io_loop_destroy(&ioloop);
 
-	doveadm_print_deinit();
+	if ((cctx.cmd->flags & CMD_FLAG_NO_PRINT) == 0)
+		doveadm_print_deinit();
 	if (o_stream_nfinish(doveadm_print_ostream)<0) {
 		i_info("Error writing output in command %s: %s",
 		       conn->cmd->name,
diff --git a/src/doveadm/doveadm-cmd.h b/src/doveadm/doveadm-cmd.h
index 6e2d2b878a1..db044979524 100644
--- a/src/doveadm/doveadm-cmd.h
+++ b/src/doveadm/doveadm-cmd.h
@@ -30,7 +30,8 @@ typedef enum {
 
 typedef enum {
 	CMD_FLAG_NONE			= 0x0,
-	CMD_FLAG_HIDDEN			= 0x1
+	CMD_FLAG_HIDDEN			= 0x1,
+	CMD_FLAG_NO_PRINT		= 0x2,
 } doveadm_cmd_flag_t;
 
 struct doveadm_cmd_param {

From 418280f5323151304327f067d3d5ef8cd9724a28 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Tue, 31 May 2016 22:22:37 +0300
Subject: [PATCH 415/450] doveadm: Implement user and auth cache flush to
 server

---
 src/doveadm/Makefile.am           |   1 +
 src/doveadm/doveadm-auth-server.c | 502 ++++++++++++++++++++++++++++++
 src/doveadm/doveadm-cmd.c         |   1 -
 src/doveadm/doveadm-cmd.h         |   1 +
 src/doveadm/doveadm.c             |   2 +-
 src/doveadm/main.c                |   1 +
 6 files changed, 506 insertions(+), 2 deletions(-)
 create mode 100644 src/doveadm/doveadm-auth-server.c

diff --git a/src/doveadm/Makefile.am b/src/doveadm/Makefile.am
index 785dc4010a6..eaf2726bbde 100644
--- a/src/doveadm/Makefile.am
+++ b/src/doveadm/Makefile.am
@@ -136,6 +136,7 @@ doveadm_SOURCES = \
 
 doveadm_server_SOURCES = \
 	$(common) \
+	doveadm-auth-server.c \
 	client-connection.c \
 	client-connection-http.c \
 	doveadm-print-server.c \
diff --git a/src/doveadm/doveadm-auth-server.c b/src/doveadm/doveadm-auth-server.c
new file mode 100644
index 00000000000..bb06220c3e0
--- /dev/null
+++ b/src/doveadm/doveadm-auth-server.c
@@ -0,0 +1,502 @@
+/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "array.h"
+#include "str.h"
+#include "var-expand.h"
+#include "wildcard-match.h"
+#include "settings-parser.h"
+#include "master-service.h"
+#include "master-service-settings.h"
+#include "auth-client.h"
+#include "auth-master.h"
+#include "auth-server-connection.h"
+#include "master-auth.h"
+#include "master-login-auth.h"
+#include "mail-storage-service.h"
+#include "mail-user.h"
+#include "ostream.h"
+#include "json-parser.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+
+#include <stdio.h>
+#include <unistd.h>
+
+struct authtest_input {
+	pool_t pool;
+	const char *username;
+	const char *master_user;
+	const char *password;
+	struct auth_user_info info;
+	bool success;
+
+	struct auth_client_request *request;
+	struct master_auth_request master_auth_req;
+
+	unsigned int auth_id;
+	unsigned int auth_pid;
+	const char *auth_cookie;
+
+};
+
+static struct auth_master_connection *
+doveadm_get_auth_master_conn(const char *auth_socket_path)
+{
+	enum auth_master_flags flags = 0;
+
+	if (doveadm_debug)
+		flags |= AUTH_MASTER_FLAG_DEBUG;
+	return auth_master_init(auth_socket_path, flags);
+}
+
+static int
+cmd_user_input(struct auth_master_connection *conn,
+	       const struct authtest_input *input,
+	       const char *show_field, bool userdb)
+{
+	const char *lookup_name = userdb ? "userdb lookup" : "passdb lookup";
+	pool_t pool;
+	const char *updated_username = NULL, *const *fields, *p;
+	int ret;
+
+	pool = pool_alloconly_create("auth master lookup", 1024);
+
+	if (userdb) {
+		ret = auth_master_user_lookup(conn, input->username, &input->info,
+					      pool, &updated_username, &fields);
+	} else {
+		ret = auth_master_pass_lookup(conn, input->username, &input->info,
+					      pool, &fields);
+	}
+	if (ret < 0) {
+		const char *msg;
+		if (fields[0] == NULL) {
+			msg = t_strdup_printf("\"error\":\"%s failed\"",
+					      lookup_name);
+		} else {
+			msg = t_strdup_printf("\"error\":\"%s failed: %s\"",
+					      lookup_name,
+					      fields[0]);
+		}
+		o_stream_nsend_str(doveadm_print_ostream, msg);
+		ret = -1;
+	} else if (ret == 0) {
+		o_stream_nsend_str(doveadm_print_ostream,
+			t_strdup_printf("\"error\":\"%s: user doesn't exist\"",
+				lookup_name));
+	} else if (show_field != NULL) {
+		unsigned int show_field_len = strlen(show_field);
+		string_t *json_field = t_str_new(show_field_len+1);
+		json_append_escaped(json_field, show_field);
+		o_stream_nsend_str(doveadm_print_ostream, t_strdup_printf("\"%s\":", str_c(json_field)));
+		for (; *fields; fields++) {
+			if (strncmp(*fields, show_field, show_field_len) == 0 &&
+			    (*fields)[show_field_len] == '=') {
+				string_t *jsonval = t_str_new(32);
+				json_append_escaped(jsonval, *fields + show_field_len + 1);
+				o_stream_nsend_str(doveadm_print_ostream, "\"");
+				o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval));
+				o_stream_nsend_str(doveadm_print_ostream, "\"");
+			}
+		}
+	} else {
+		string_t *jsonval = t_str_new(64);
+		o_stream_nsend_str(doveadm_print_ostream, "\"source\":\"");
+		o_stream_nsend_str(doveadm_print_ostream, userdb ? "userdb\"" : "passdb\"");
+
+		if (updated_username != NULL) {
+			o_stream_nsend_str(doveadm_print_ostream, ",\"updated_username\":\"");
+			str_truncate(jsonval, 0);
+			json_append_escaped(jsonval, updated_username);
+			o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval));
+			o_stream_nsend_str(doveadm_print_ostream, "\"");
+		}
+		for (; *fields; fields++) {
+			const char *field = *fields;
+			if (*field == '\0') continue;
+			p = strchr(*fields, '=');
+			str_truncate(jsonval, 0);
+			if (p != NULL) {
+				field = t_strcut(*fields, '=');
+			}
+			str_truncate(jsonval, 0);
+			json_append_escaped(jsonval, field);
+			o_stream_nsend_str(doveadm_print_ostream, ",\"");
+			o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval));
+			o_stream_nsend_str(doveadm_print_ostream, "\":");
+			if (p != NULL) {
+				str_truncate(jsonval, 0);
+				json_append_escaped(jsonval, p+1);
+				o_stream_nsend_str(doveadm_print_ostream, "\"");
+				o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval));
+				o_stream_nsend_str(doveadm_print_ostream, "\"");
+			} else {
+				o_stream_nsend_str(doveadm_print_ostream, "true");
+			}
+		}
+	}
+	return ret;
+}
+
+static void auth_user_info_parse(struct auth_user_info *info, const char *arg)
+{
+	if (strncmp(arg, "service=", 8) == 0)
+		info->service = arg + 8;
+	else if (strncmp(arg, "lip=", 4) == 0) {
+		if (net_addr2ip(arg + 4, &info->local_ip) < 0)
+			i_fatal("lip: Invalid ip");
+	} else if (strncmp(arg, "rip=", 4) == 0) {
+		if (net_addr2ip(arg + 4, &info->remote_ip) < 0)
+			i_fatal("rip: Invalid ip");
+	} else if (strncmp(arg, "lport=", 6) == 0) {
+		if (net_str2port(arg + 6, &info->local_port) < 0)
+			i_fatal("lport: Invalid port number");
+	} else if (strncmp(arg, "rport=", 6) == 0) {
+		if (net_str2port(arg + 6, &info->remote_port) < 0)
+			i_fatal("rport: Invalid port number");
+	} else {
+		i_fatal("Unknown -x argument: %s", arg);
+	}
+}
+
+static void
+cmd_user_list(struct auth_master_connection *conn,
+	      const struct authtest_input *input,
+	      char *const *users)
+{
+	struct auth_master_user_list_ctx *ctx;
+	const char *username, *user_mask = "*";
+	unsigned int i;
+
+	if (users[0] != NULL && users[1] == NULL)
+		user_mask = users[0];
+
+	ctx = auth_master_user_list_init(conn, user_mask, &input->info);
+	while ((username = auth_master_user_list_next(ctx)) != NULL) {
+		for (i = 0; users[i] != NULL; i++) {
+			if (wildcard_match_icase(username, users[i]))
+				break;
+		}
+		if (users[i] != NULL)
+			printf("%s\n", username);
+	}
+	if (auth_master_user_list_deinit(&ctx) < 0)
+		i_fatal("user listing failed");
+}
+
+static void cmd_auth_cache_flush(int argc, char *argv[])
+{
+	const char *master_socket_path = NULL;
+	struct auth_master_connection *conn;
+	unsigned int count;
+	int c;
+
+	while ((c = getopt(argc, argv, "a:")) > 0) {
+		switch (c) {
+		case 'a':
+			master_socket_path = optarg;
+			break;
+		default:
+			doveadm_exit_code = EX_USAGE;
+			return;
+		}
+	}
+	argv += optind;
+
+	if (master_socket_path == NULL) {
+		master_socket_path = t_strconcat(doveadm_settings->base_dir,
+						 "/auth-master", NULL);
+	}
+
+	conn = doveadm_get_auth_master_conn(master_socket_path);
+	if (auth_master_cache_flush(conn, (void *)argv, &count) < 0) {
+		i_error("Cache flush failed");
+		doveadm_exit_code = EX_TEMPFAIL;
+	} else {
+		doveadm_print_init("formatted");
+		doveadm_print_formatted_set_format("%{entries} cache entries flushed\n");
+		doveadm_print_header_simple("entries");
+		doveadm_print_num(count);
+	}
+	auth_master_deinit(&conn);
+}
+
+static void cmd_user_mail_input_field(const char *key, const char *value,
+				      const char *show_field)
+{
+	string_t *jvalue = t_str_new(128);
+	if (show_field != NULL && strcmp(show_field, key) != 0) return;
+	json_append_escaped(jvalue, key);
+	o_stream_nsend_str(doveadm_print_ostream, "\"");
+	o_stream_nsend_str(doveadm_print_ostream, str_c(jvalue));
+	o_stream_nsend_str(doveadm_print_ostream, "\":\"");
+	str_truncate(jvalue, 0);
+	json_append_escaped(jvalue, value);
+	o_stream_nsend_str(doveadm_print_ostream, str_c(jvalue));
+	o_stream_nsend_str(doveadm_print_ostream, "\"");
+}
+
+static void
+cmd_user_mail_print_fields(const struct authtest_input *input,
+			   struct mail_user *user,
+			   const char *const *userdb_fields,
+			   const char *show_field)
+{
+	const struct mail_storage_settings *mail_set;
+	const char *key, *value;
+	unsigned int i;
+
+	if (strcmp(input->username, user->username) != 0) {
+		cmd_user_mail_input_field("user", user->username, show_field);
+		o_stream_nsend_str(doveadm_print_ostream, ",");
+	}
+	cmd_user_mail_input_field("uid", user->set->mail_uid, show_field);
+	o_stream_nsend_str(doveadm_print_ostream, ",");
+	cmd_user_mail_input_field("gid", user->set->mail_gid, show_field);
+	o_stream_nsend_str(doveadm_print_ostream, ",");
+	cmd_user_mail_input_field("home", user->set->mail_home, show_field);
+
+	mail_set = mail_user_set_get_storage_set(user);
+	o_stream_nsend_str(doveadm_print_ostream, ",");
+	cmd_user_mail_input_field("mail", mail_set->mail_location, show_field);
+
+	if (userdb_fields != NULL) {
+		for (i = 0; userdb_fields[i] != NULL; i++) {
+			value = strchr(userdb_fields[i], '=');
+			if (value != NULL)
+				key = t_strdup_until(userdb_fields[i], value++);
+			else {
+				key = userdb_fields[i];
+				value = "";
+			}
+			if (strcmp(key, "uid") != 0 &&
+			    strcmp(key, "gid") != 0 &&
+			    strcmp(key, "home") != 0 &&
+			    strcmp(key, "mail") != 0 &&
+			    *key != '\0') {
+				o_stream_nsend_str(doveadm_print_ostream, ",");
+				cmd_user_mail_input_field(key, value, show_field);
+			}
+		}
+	}
+}
+
+static int
+cmd_user_mail_input(struct mail_storage_service_ctx *storage_service,
+		    const struct authtest_input *input,
+		    const char *show_field, const char *expand_field)
+{
+	struct mail_storage_service_input service_input;
+	struct mail_storage_service_user *service_user;
+	struct mail_user *user;
+	const char *error, *const *userdb_fields;
+	pool_t pool;
+	int ret;
+
+	memset(&service_input, 0, sizeof(service_input));
+	service_input.module = "mail";
+	service_input.service = input->info.service;
+	service_input.username = input->username;
+	service_input.local_ip = input->info.local_ip;
+	service_input.local_port = input->info.local_port;
+	service_input.remote_ip = input->info.remote_ip;
+	service_input.remote_port = input->info.remote_port;
+	service_input.debug = input->info.debug;
+
+	pool = pool_alloconly_create("userdb fields", 1024);
+	mail_storage_service_save_userdb_fields(storage_service, pool,
+						&userdb_fields);
+
+	if ((ret = mail_storage_service_lookup_next(storage_service, &service_input,
+						    &service_user, &user,
+						    &error)) <= 0) {
+		pool_unref(&pool);
+		if (ret < 0)
+			return -1;
+		string_t *username = t_str_new(32);
+		json_append_escaped(username, input->username);
+		o_stream_nsend_str(doveadm_print_ostream,
+			t_strdup_printf("\"error\":\"userdb lookup: user %s doesn't exist\"", str_c(username))
+		);
+		return 0;
+	}
+
+	if (expand_field == NULL)
+		cmd_user_mail_print_fields(input, user, userdb_fields, show_field);
+	else {
+		string_t *str = t_str_new(128);
+		var_expand_with_funcs(str, expand_field,
+				      mail_user_var_expand_table(user),
+				      mail_user_var_expand_func_table, user);
+		string_t *value = t_str_new(128);
+		json_append_escaped(value, expand_field);
+		o_stream_nsend_str(doveadm_print_ostream, "\"");
+		o_stream_nsend_str(doveadm_print_ostream, str_c(value));
+		o_stream_nsend_str(doveadm_print_ostream, "\":\"");
+		str_truncate(value, 0);
+		json_append_escaped(value, str_c(str));
+		o_stream_nsend_str(doveadm_print_ostream, str_c(value));
+		o_stream_nsend_str(doveadm_print_ostream, "\"");
+
+	}
+
+	mail_user_unref(&user);
+	mail_storage_service_user_free(&service_user);
+	pool_unref(&pool);
+	return 1;
+}
+
+static void cmd_user_ver2(struct doveadm_cmd_context *cctx)
+{
+	const char * const *optval;
+
+	const char *auth_socket_path = NULL;
+	struct auth_master_connection *conn;
+	struct authtest_input input;
+	const char *show_field = NULL, *expand_field = NULL;
+	struct mail_storage_service_ctx *storage_service = NULL;
+	bool have_wildcards, userdb_only = FALSE, first = TRUE;
+	int ret;
+
+	if (!doveadm_cmd_param_str(cctx, "socket-path", &auth_socket_path))
+		auth_socket_path = doveadm_settings->auth_socket_path;
+
+	(void)doveadm_cmd_param_str(cctx, "expand-field", &expand_field);
+	(void)doveadm_cmd_param_str(cctx, "field", &show_field);
+	(void)doveadm_cmd_param_bool(cctx, "userdb-only", &userdb_only);
+
+	if (doveadm_cmd_param_array(cctx, "auth-info", &optval))
+		for(;*optval != NULL; optval++)
+			auth_user_info_parse(&input.info, *optval);
+
+	if (!doveadm_cmd_param_array(cctx, "user-mask", &optval)) {
+		doveadm_exit_code = EX_USAGE;
+		i_error("No user(s) specified");
+		return;
+	}
+
+	if (expand_field != NULL && userdb_only) {
+		i_error("-e can't be used with -u");
+		doveadm_exit_code = EX_USAGE;
+		return;
+	}
+	if (expand_field != NULL && show_field != NULL) {
+		i_error("-e can't be used with -f");
+		doveadm_exit_code = EX_USAGE;
+		return;
+	}
+
+	conn = doveadm_get_auth_master_conn(auth_socket_path);
+
+	have_wildcards = FALSE;
+
+	for(const char *const *val = optval; *val != NULL; val++) {
+		if (strchr(*val, '*') != NULL ||
+		    strchr(*val, '?') != NULL) {
+			have_wildcards = TRUE;
+			break;
+		}
+	}
+
+	if (have_wildcards) {
+		cmd_user_list(conn, &input, (char*const*)optval);
+		auth_master_deinit(&conn);
+		return;
+	}
+
+	if (!userdb_only) {
+		storage_service = mail_storage_service_init(master_service, NULL,
+			MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP |
+			MAIL_STORAGE_SERVICE_FLAG_NO_CHDIR |
+			MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT |
+			MAIL_STORAGE_SERVICE_FLAG_NO_PLUGINS |
+			MAIL_STORAGE_SERVICE_FLAG_NO_NAMESPACES |
+			MAIL_STORAGE_SERVICE_FLAG_NO_RESTRICT_ACCESS);
+		mail_storage_service_set_auth_conn(storage_service, conn);
+		conn = NULL;
+	}
+
+	string_t *json = t_str_new(64);
+	o_stream_nsend_str(doveadm_print_ostream, "{");
+
+	input.info.local_ip = cctx->local_ip;
+	input.info.local_port = cctx->local_port;
+	input.info.remote_ip = cctx->remote_ip;
+	input.info.remote_port = cctx->remote_port;
+
+	for(const char *const *val = optval; *val != NULL; val++) {
+		str_truncate(json, 0);
+		json_append_escaped(json, *val);
+
+		input.username = *val;
+		if (first)
+			first = FALSE;
+		else
+			o_stream_nsend_str(doveadm_print_ostream, ",");
+
+		o_stream_nsend_str(doveadm_print_ostream, "\"");
+		o_stream_nsend_str(doveadm_print_ostream, str_c(json));
+		o_stream_nsend_str(doveadm_print_ostream, "\"");
+		o_stream_nsend_str(doveadm_print_ostream, ":{");
+
+		ret = !userdb_only ?
+			cmd_user_mail_input(storage_service, &input, show_field, expand_field) :
+			cmd_user_input(conn, &input, show_field, TRUE);
+
+		o_stream_nsend_str(doveadm_print_ostream, "}");
+
+		switch (ret) {
+		case -1:
+			doveadm_exit_code = EX_TEMPFAIL;
+			break;
+		case 0:
+			doveadm_exit_code = EX_NOUSER;
+			break;
+		}
+	}
+
+	o_stream_nsend_str(doveadm_print_ostream,"}");
+
+	if (storage_service != NULL)
+		mail_storage_service_deinit(&storage_service);
+	if (conn != NULL)
+		auth_master_deinit(&conn);
+}
+
+static
+struct doveadm_cmd_ver2 doveadm_cmd_auth_server[] = {
+{
+	.name = "auth cache flush",
+	.old_cmd = cmd_auth_cache_flush,
+	.usage = "[-a <master socket path>] [<user> [...]]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+	.name = "user",
+	.cmd = cmd_user_ver2,
+	.usage = "[-a <userdb socket path>] [-x <auth info>] [-f field] [-e <value>] [-u] <user mask> [...]",
+	.flags = CMD_FLAG_NO_PRINT,
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('x', "auth-info", CMD_PARAM_ARRAY, 0)
+DOVEADM_CMD_PARAM('f', "field", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('e', "expand-field", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('u', "userdb-only", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "user-mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+}
+};
+
+void doveadm_register_auth_server_commands(void)
+{
+	unsigned int i;
+
+	for (i = 0; i < N_ELEMENTS(doveadm_cmd_auth_server); i++) {
+		doveadm_cmd_register_ver2(&doveadm_cmd_auth_server[i]);
+	}
+}
diff --git a/src/doveadm/doveadm-cmd.c b/src/doveadm/doveadm-cmd.c
index 0bb8dd408b9..46bb92bd6bf 100644
--- a/src/doveadm/doveadm-cmd.c
+++ b/src/doveadm/doveadm-cmd.c
@@ -173,7 +173,6 @@ void doveadm_cmds_init(void)
 	for (i = 0; i < N_ELEMENTS(doveadm_commands_ver2); i++)
 		doveadm_cmd_register_ver2(doveadm_commands_ver2[i]);
 
-	doveadm_register_auth_commands();
 	doveadm_register_director_commands();
 	doveadm_register_instance_commands();
 	doveadm_register_mount_commands();
diff --git a/src/doveadm/doveadm-cmd.h b/src/doveadm/doveadm-cmd.h
index db044979524..3c677daaa7b 100644
--- a/src/doveadm/doveadm-cmd.h
+++ b/src/doveadm/doveadm-cmd.h
@@ -101,6 +101,7 @@ doveadm_cmd_find_with_args(const char *cmd_name, int *argc,
 			   const char *const *argv[]);
 
 void doveadm_register_auth_commands(void);
+void doveadm_register_auth_server_commands(void);
 void doveadm_register_director_commands(void);
 void doveadm_register_proxy_commands(void);
 void doveadm_register_log_commands(void);
diff --git a/src/doveadm/doveadm.c b/src/doveadm/doveadm.c
index 677047e0083..db15a3cc1ed 100644
--- a/src/doveadm/doveadm.c
+++ b/src/doveadm/doveadm.c
@@ -333,7 +333,7 @@ int main(int argc, char *argv[])
 	doveadm_cmds_init();
 	for (i = 0; i < N_ELEMENTS(doveadm_cmdline_commands); i++)
 		doveadm_register_cmd(doveadm_cmdline_commands[i]);
-
+	doveadm_register_auth_commands();
 	doveadm_cmd_register_ver2(&doveadm_cmd_stats_top_ver2);
 
 	if (cmd_name != NULL && (quick_init ||
diff --git a/src/doveadm/main.c b/src/doveadm/main.c
index dca58467425..744d58c4cee 100644
--- a/src/doveadm/main.c
+++ b/src/doveadm/main.c
@@ -73,6 +73,7 @@ static void main_init(void)
 
 	doveadm_http_server_init();
 	doveadm_cmds_init();
+	doveadm_register_auth_server_commands();
 	doveadm_dump_init();
 	doveadm_mail_init();
 	dict_drivers_register_builtin();

From 3626987638bd377c37474d6f95a26da24cdf4ac7 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Fri, 3 Jun 2016 17:30:58 +0300
Subject: [PATCH 416/450] doveadm-server: Do not crash if empty data in
 authorization

---
 src/doveadm/client-connection-http.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/doveadm/client-connection-http.c b/src/doveadm/client-connection-http.c
index 076f6bee750..63716faff9e 100644
--- a/src/doveadm/client-connection-http.c
+++ b/src/doveadm/client-connection-http.c
@@ -659,13 +659,13 @@ doveadm_http_server_authorize_request(struct client_connection_http *conn)
 			string_t *b64_value = str_new(conn->client.pool, 32);
 			char *value = p_strdup_printf(conn->client.pool, "doveadm:%s", conn->client.set->doveadm_password);
 			base64_encode(value, strlen(value), b64_value);
-			if (strcmp(creds.data, str_c(b64_value)) == 0) auth = TRUE;
+			if (creds.data != NULL && strcmp(creds.data, str_c(b64_value)) == 0) auth = TRUE;
 			else i_error("Invalid authentication attempt to HTTP API");
 		}
 		else if (strcasecmp(creds.scheme, "X-Dovecot-API") == 0 && doveadm_settings->doveadm_api_key[0] != '\0') {
 			string_t *b64_value = str_new(conn->client.pool, 32);
 			base64_encode(doveadm_settings->doveadm_api_key, strlen(doveadm_settings->doveadm_api_key), b64_value);
-			if (strcmp(creds.data, str_c(b64_value)) == 0) auth = TRUE;
+			if (creds.data != NULL && strcmp(creds.data, str_c(b64_value)) == 0) auth = TRUE;
 			else i_error("Invalid authentication attempt to HTTP API");
 		}
 		else i_error("Unsupported authentication scheme to HTTP API");

From 0649b7a1656bd98d95cdf40a98d47cff9c8de9f8 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 18:31:21 +0300
Subject: [PATCH 417/450] maildir: Fixed updating filenames in existing uidlist

Broken by 042668c0c.

This could have caused errors like:

 - maildir_file_do(...): Filename keeps changing
 - Expunged message reappeared, giving a new UID
---
 src/lib-storage/index/maildir/maildir-uidlist.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/lib-storage/index/maildir/maildir-uidlist.c b/src/lib-storage/index/maildir/maildir-uidlist.c
index 6208ed29cbe..bed17e5d2e5 100644
--- a/src/lib-storage/index/maildir/maildir-uidlist.c
+++ b/src/lib-storage/index/maildir/maildir-uidlist.c
@@ -1706,6 +1706,8 @@ maildir_uidlist_sync_next_partial(struct maildir_uidlist_sync_ctx *ctx,
 		uidlist->change_counter++;
 
 		hash_table_insert(uidlist->files, rec->filename, rec);
+	} else if (strcmp(rec->filename, filename) != 0) {
+		rec->filename = p_strdup(uidlist->record_pool, filename);
 	}
 	if (uid != 0) {
 		if (rec->uid != uid && rec->uid != (uint32_t)-1) {
@@ -1800,6 +1802,8 @@ int maildir_uidlist_sync_next_uid(struct maildir_uidlist_sync_ctx *ctx,
 		   to check for duplicates. */
 		rec->flags &= ~(MAILDIR_UIDLIST_REC_FLAG_NEW_DIR |
 				MAILDIR_UIDLIST_REC_FLAG_MOVED);
+		if (strcmp(rec->filename, filename) != 0)
+			rec->filename = p_strdup(ctx->record_pool, filename);
 	} else {
 		old_rec = hash_table_lookup(uidlist->files, filename);
 		i_assert(old_rec != NULL || UIDLIST_IS_LOCKED(uidlist));

From 26e22eaa0f6909c0e2e33dc0e59e0d08a3f319c2 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 18:33:48 +0300
Subject: [PATCH 418/450] maildir: Code comment update

---
 src/lib-storage/index/maildir/maildir-sync.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/lib-storage/index/maildir/maildir-sync.c b/src/lib-storage/index/maildir/maildir-sync.c
index 1b80e41638f..9ad66093eb1 100644
--- a/src/lib-storage/index/maildir/maildir-sync.c
+++ b/src/lib-storage/index/maildir/maildir-sync.c
@@ -972,8 +972,9 @@ maildir_sync_context(struct maildir_sync_context *ctx, bool forced,
 			/* UID is expunged */
 			*find_uid = 0;
 		} else if ((flags & MAILDIR_UIDLIST_REC_FLAG_NONSYNCED) == 0) {
-			/* we didn't find it, possibly expunged? */
 			*find_uid = 0;
+		} else {
+			/* we didn't find it, possibly expunged? */
 		}
 	}
 

From 390d6d596cdc6d59ee3013cec91bfecf638cbef1 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 18:34:07 +0300
Subject: [PATCH 419/450] maildir: Improved "Filename keeps changing" error
 handling/logging

---
 src/lib-storage/index/maildir/maildir-util.c | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/lib-storage/index/maildir/maildir-util.c b/src/lib-storage/index/maildir/maildir-util.c
index e527989792c..e3b76051993 100644
--- a/src/lib-storage/index/maildir/maildir-util.c
+++ b/src/lib-storage/index/maildir/maildir-util.c
@@ -117,18 +117,26 @@ static int maildir_file_do_try(struct maildir_mailbox *mbox, uint32_t uid,
 }
 
 static int do_racecheck(struct maildir_mailbox *mbox, const char *path,
-			void *context ATTR_UNUSED)
+			void *context)
 {
+	const uint32_t *uidp = context;
 	struct stat st;
+	int ret;
 
-	if (lstat(path, &st) == 0 && (st.st_mode & S_IFMT) == S_IFLNK) {
+	ret = lstat(path, &st);
+	if (ret == 0 && (st.st_mode & S_IFMT) == S_IFLNK) {
 		/* most likely a symlink pointing to a nonexistent file */
 		mail_storage_set_critical(&mbox->storage->storage,
-			"Maildir: Symlink destination doesn't exist: %s", path);
+			"Maildir: Symlink destination doesn't exist for UID=%u: %s", *uidp, path);
 		return -2;
+	} else if (ret < 0 && errno != ENOENT) {
+		mail_storage_set_critical(&mbox->storage->storage,
+			"lstat(%s) failed: %m", path);
+		return -1;
 	} else {
+		/* success or ENOENT, either way we're done */
 		mail_storage_set_critical(&mbox->storage->storage,
-			"maildir_file_do(%s): Filename keeps changing", path);
+			"maildir_file_do(%s): Filename keeps changing for UID=%u", path, *uidp);
 		return -1;
 	}
 }
@@ -160,7 +168,7 @@ int maildir_file_do(struct maildir_mailbox *mbox, uint32_t uid,
 	}
 
 	if (i == MAILDIR_RESYNC_RETRY_COUNT) T_BEGIN {
-		ret = maildir_file_do_try(mbox, uid, do_racecheck, context);
+		ret = maildir_file_do_try(mbox, uid, do_racecheck, &uid);
 	} T_END;
 
 	return ret == -2 ? 0 : ret;

From 6b1428ff188c594d6e7e578f994dac659caad64d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 18:49:57 +0300
Subject: [PATCH 420/450] virtual: Fixed error handling when matching mailboxes
 by metadata.

---
 src/plugins/virtual/virtual-config.c | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/plugins/virtual/virtual-config.c b/src/plugins/virtual/virtual-config.c
index 6fe588e8f4d..270c1881120 100644
--- a/src/plugins/virtual/virtual-config.c
+++ b/src/plugins/virtual/virtual-config.c
@@ -348,13 +348,11 @@ static int virtual_config_box_metadata_match(struct mailbox *box,
 
 	imtrans = imap_metadata_transaction_begin(box);
 	ret = imap_metadata_get(imtrans, bbox->metadata_entry, &value);
-	if (ret < 0) {
+	if (ret < 0)
 		*error_r = t_strdup(imap_metadata_transaction_get_last_error(imtrans, NULL));
-		return -1;
-	}
 	if (ret > 0)
 		ret = wildcard_match(value.value, bbox->metadata_value) ? 1 : 0;
-	if (bbox->negative_match)
+	if (ret >= 0 && bbox->negative_match)
 		ret = ret > 0 ? 0 : 1;
 	(void)imap_metadata_transaction_commit(&imtrans, NULL, NULL);
 	return ret;
@@ -445,7 +443,7 @@ static int virtual_config_expand_wildcards(struct virtual_parse_context *ctx,
 		*error_r = mailbox_list_get_last_error(user->namespaces->list, NULL);
 		return -1;
 	}
-	return 0;
+	return ret < 0 ? -1 : 0;
 }
 
 static void virtual_config_search_args_dup(struct virtual_mailbox *mbox)

From 127b836fd82f421767da3bf843fca55f39f1b109 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 19:12:58 +0300
Subject: [PATCH 421/450] lib-storage: Fixed search arg initialization tracking
 for INTHREAD

---
 src/lib-storage/mail-search.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/lib-storage/mail-search.c b/src/lib-storage/mail-search.c
index 4ffbba09b65..3c7cd4801cd 100644
--- a/src/lib-storage/mail-search.c
+++ b/src/lib-storage/mail-search.c
@@ -105,6 +105,7 @@ void mail_search_arg_init(struct mail_search_args *args,
 				thread_args->pool = args->pool;
 				thread_args->args = arg->value.subargs;
 				thread_args->simplified = TRUE;
+				thread_args->init_refcount = 1;
 				/* simplification should have unnested all
 				   inthreads, so we'll assume that
 				   have_inthreads=FALSE */

From fde7b8a03bf91cfa5bb7ca3e84545386243fa0d2 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 28 Jun 2016 23:38:11 +0300
Subject: [PATCH 422/450] master: Added support for stopping specific services.

We need to have a per-service fd for detecting a dead master.
---
 src/master/service-monitor.c | 26 ++++++++++++--------------
 src/master/service-process.c |  2 +-
 src/master/service.c         |  4 ++--
 src/master/service.h         |  4 ++--
 4 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/src/master/service-monitor.c b/src/master/service-monitor.c
index bf4e9312410..fb4f64f5b25 100644
--- a/src/master/service-monitor.c
+++ b/src/master/service-monitor.c
@@ -452,11 +452,6 @@ void services_monitor_start(struct service_list *service_list)
 		return;
 	service_anvil_monitor_start(service_list);
 
-	if (pipe(service_list->master_dead_pipe_fd) < 0)
-		i_error("pipe() failed: %m");
-	fd_close_on_exec(service_list->master_dead_pipe_fd[0], TRUE);
-	fd_close_on_exec(service_list->master_dead_pipe_fd[1], TRUE);
-
 	array_foreach(&service_list->services, services) {
 		struct service *service = *services;
 
@@ -464,6 +459,14 @@ void services_monitor_start(struct service_list *service_list)
 			if (service_login_create_notify_fd(service) < 0)
 				continue;
 		}
+		if (service->master_dead_pipe_fd[0] == -1) {
+			if (pipe(service->master_dead_pipe_fd) < 0) {
+				service_error(service, "pipe() failed: %m");
+				continue;
+			}
+			fd_close_on_exec(service->master_dead_pipe_fd[0], TRUE);
+			fd_close_on_exec(service->master_dead_pipe_fd[1], TRUE);
+		}
 		if (service->status_fd[0] == -1) {
 			/* we haven't yet created status pipe */
 			if (pipe(service->status_fd) < 0) {
@@ -519,6 +522,10 @@ void service_monitor_stop(struct service *service)
 			service->status_fd[i] = -1;
 		}
 	}
+	if (service->master_dead_pipe_fd[0] != -1) {
+		i_close_fd(&service->master_dead_pipe_fd[0]);
+		i_close_fd(&service->master_dead_pipe_fd[1]);
+	}
 	if (service->login_notify_fd != -1) {
 		if (close(service->login_notify_fd) < 0) {
 			service_error(service,
@@ -561,15 +568,6 @@ void services_monitor_stop(struct service_list *service_list, bool wait)
 {
 	struct service *const *services;
 
-	if (service_list->master_dead_pipe_fd[0] != -1) {
-		if (close(service_list->master_dead_pipe_fd[0]) < 0)
-			i_error("close(master dead pipe) failed: %m");
-		if (close(service_list->master_dead_pipe_fd[1]) < 0)
-			i_error("close(master dead pipe) failed: %m");
-		service_list->master_dead_pipe_fd[0] = -1;
-		service_list->master_dead_pipe_fd[1] = -1;
-	}
-
 	if (wait) {
 		/* we've notified all children that the master is dead.
 		   now wait for the children to either die or to tell that
diff --git a/src/master/service-process.c b/src/master/service-process.c
index d0f1df69a12..7a24c58394c 100644
--- a/src/master/service-process.c
+++ b/src/master/service-process.c
@@ -133,7 +133,7 @@ service_dup_fds(struct service *service)
 	}
 	dup2_append(&dups, service->status_fd[1], MASTER_STATUS_FD);
 	if (service->type != SERVICE_TYPE_ANVIL) {
-		dup2_append(&dups, service->list->master_dead_pipe_fd[1],
+		dup2_append(&dups, service->master_dead_pipe_fd[1],
 			    MASTER_DEAD_FD);
 	} else {
 		dup2_append(&dups, global_master_dead_pipe_fd[1],
diff --git a/src/master/service.c b/src/master/service.c
index d9f49d6bb72..d175b0c4b95 100644
--- a/src/master/service.c
+++ b/src/master/service.c
@@ -283,6 +283,8 @@ service_create(pool_t pool, const struct service_settings *set,
 	service->log_fd[1] = -1;
 	service->status_fd[0] = -1;
 	service->status_fd[1] = -1;
+	service->master_dead_pipe_fd[0] = -1;
+	service->master_dead_pipe_fd[1] = -1;
 	service->log_process_internal_fd = -1;
 	service->login_notify_fd = -1;
 
@@ -424,8 +426,6 @@ services_create_real(const struct master_settings *set, pool_t pool,
 	service_list->set = set;
 	service_list->master_log_fd[0] = -1;
 	service_list->master_log_fd[1] = -1;
-	service_list->master_dead_pipe_fd[0] = -1;
-	service_list->master_dead_pipe_fd[1] = -1;
 
 	service_settings = array_get(&set->services, &count);
 	p_array_init(&service_list->services, pool, count);
diff --git a/src/master/service.h b/src/master/service.h
index 79aa65f9ef0..3887e52499a 100644
--- a/src/master/service.h
+++ b/src/master/service.h
@@ -85,6 +85,8 @@ struct service {
 	int status_fd[2];
 	struct io *io_status;
 
+	int master_dead_pipe_fd[2];
+
 	unsigned int throttle_secs;
 	time_t exit_failure_last;
 	unsigned int exit_failures_in_sec;
@@ -139,8 +141,6 @@ struct service_list {
 	int master_log_fd[2];
 	struct service_process_notify *log_byes;
 
-	int master_dead_pipe_fd[2];
-
 	ARRAY(struct service *) services;
 
 	unsigned int destroying:1;

From 976683c774f42569fc7b3edcbc6068808e2c275f Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 28 Jun 2016 23:38:48 +0300
Subject: [PATCH 423/450] master: Added service_monitor_stop_close() to stops
 service and close its listeners.

---
 src/master/service-monitor.c | 14 ++++++++++++++
 src/master/service-monitor.h |  1 +
 2 files changed, 15 insertions(+)

diff --git a/src/master/service-monitor.c b/src/master/service-monitor.c
index fb4f64f5b25..a90c4ebe8ce 100644
--- a/src/master/service-monitor.c
+++ b/src/master/service-monitor.c
@@ -543,6 +543,20 @@ void service_monitor_stop(struct service *service)
 		timeout_remove(&service->to_prefork);
 }
 
+void service_monitor_stop_close(struct service *service)
+{
+	struct service_listener *const *listeners;
+
+	service_monitor_stop(service);
+
+	array_foreach(&service->listeners, listeners) {
+		struct service_listener *l = *listeners;
+
+		if (l->fd != -1)
+			i_close_fd(&l->fd);
+	}
+}
+
 static void services_monitor_wait(struct service_list *service_list)
 {
 	struct service *const *servicep;
diff --git a/src/master/service-monitor.h b/src/master/service-monitor.h
index 848cb3c1e0e..7dd1c4db91f 100644
--- a/src/master/service-monitor.h
+++ b/src/master/service-monitor.h
@@ -11,6 +11,7 @@ void services_monitor_stop(struct service_list *service_list, bool wait);
 void services_monitor_reap_children(void);
 
 void service_monitor_stop(struct service *service);
+void service_monitor_stop_close(struct service *service);
 void service_monitor_listen_start(struct service *service);
 void service_monitor_listen_stop(struct service *service);
 

From e296bf525cce5d95595abb9200c28773a72412e8 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Sun, 26 Jun 2016 23:04:00 +0300
Subject: [PATCH 424/450] master: Allow stopping specific services via master
 socket.

---
 src/master/Makefile.am       |   2 +
 src/master/main.c            |   3 ++
 src/master/master-client.c   | 100 +++++++++++++++++++++++++++++++++++
 src/master/master-client.h   |   9 ++++
 src/master/service-listen.c  |  29 ++++++++++
 src/master/service-monitor.c |   5 ++
 src/master/service.h         |   4 ++
 7 files changed, 152 insertions(+)
 create mode 100644 src/master/master-client.c
 create mode 100644 src/master/master-client.h

diff --git a/src/master/Makefile.am b/src/master/Makefile.am
index 9d3ad7844f0..ef7955ff315 100644
--- a/src/master/Makefile.am
+++ b/src/master/Makefile.am
@@ -25,6 +25,7 @@ dovecot_SOURCES = \
 	capabilities-posix.c \
 	dup2-array.c \
 	main.c \
+	master-client.c \
 	master-settings.c \
 	service-anvil.c \
 	service-listen.c \
@@ -39,6 +40,7 @@ noinst_HEADERS = \
 	capabilities.h \
 	common.h \
 	dup2-array.h \
+	master-client.h \
 	master-settings.h \
 	sd-daemon.h \
 	service-anvil.h \
diff --git a/src/master/main.c b/src/master/main.c
index a98a0b4c56f..8fd4cb77d27 100644
--- a/src/master/main.c
+++ b/src/master/main.c
@@ -18,6 +18,7 @@
 #include "master-service-settings.h"
 #include "askpass.h"
 #include "capabilities.h"
+#include "master-client.h"
 #include "service.h"
 #include "service-anvil.h"
 #include "service-listen.h"
@@ -526,6 +527,7 @@ static void main_init(const struct master_settings *set)
 	create_pid_file(pidfile_path);
 	create_config_symlink(set);
 	instance_update(set);
+	master_clients_init();
 
 	services_monitor_start(services);
 }
@@ -542,6 +544,7 @@ static void global_dead_pipe_close(void)
 
 static void main_deinit(void)
 {
+	master_clients_deinit();
 	instance_update_now(instances);
 	timeout_remove(&to_instance);
 	master_instance_list_deinit(&instances);
diff --git a/src/master/master-client.c b/src/master/master-client.c
new file mode 100644
index 00000000000..32cfbc3eb4f
--- /dev/null
+++ b/src/master/master-client.c
@@ -0,0 +1,100 @@
+/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */
+
+#include "common.h"
+#include "ostream.h"
+#include "connection.h"
+#include "service.h"
+#include "service-monitor.h"
+#include "master-client.h"
+
+struct master_client {
+	struct connection conn;
+};
+
+static int
+master_client_stop(struct master_client *client, const char *const *args)
+{
+	struct service *service;
+	const char *reply = "+\n";
+
+	for (unsigned int i = 0; args[i] != NULL; i++) {
+		service = service_lookup(services, args[i]);
+		if (service == NULL)
+			reply = t_strdup_printf("-Unknown service: %s\n", args[i]);
+		else
+			service_monitor_stop_close(service);
+	}
+	o_stream_send_str(client->conn.output, reply);
+	return 1;
+}
+
+static int
+master_client_input_args(struct connection *conn, const char *const *args)
+{
+	struct master_client *client = (struct master_client *)conn;
+	const char *cmd = args[0];
+
+	if (cmd == NULL) {
+		i_error("%s: Empty command", conn->name);
+		return 0;
+	}
+	args++;
+
+	if (strcmp(cmd, "STOP") == 0)
+		return master_client_stop(client, args);
+	i_error("%s: Unknown command: %s", conn->name, cmd);
+	return -1;
+}
+
+static void master_client_destroy(struct connection *conn)
+{
+	struct master_client *client = (struct master_client *)conn;
+
+	connection_deinit(conn);
+	i_free(client);
+}
+
+static const struct connection_settings master_conn_set = {
+	.service_name_in = "master-client",
+	.service_name_out = "master-server",
+	.major_version = 1,
+	.minor_version = 0,
+
+	.input_max_size = 1024,
+	.output_max_size = 1024,
+	.client = FALSE
+};
+
+static const struct connection_vfuncs master_conn_vfuncs = {
+	.destroy = master_client_destroy,
+	.input_args = master_client_input_args
+};
+
+static struct connection_list *master_connections;
+
+void master_client_connected(struct service_list *service_list)
+{
+	struct master_client *client;
+	int fd;
+
+	fd = net_accept(service_list->master_fd, NULL, NULL);
+	if (fd < 0) {
+		if (fd == -2)
+			i_error("net_accept() failed: %m");
+		return;
+	}
+	client = i_new(struct master_client, 1);
+	connection_init_server(master_connections, &client->conn,
+			       "master-client", fd, fd);
+}
+
+void master_clients_init(void)
+{
+	master_connections = connection_list_init(&master_conn_set,
+						  &master_conn_vfuncs);
+}
+
+void master_clients_deinit(void)
+{
+	connection_list_deinit(&master_connections);
+}
diff --git a/src/master/master-client.h b/src/master/master-client.h
new file mode 100644
index 00000000000..7e62edee8a6
--- /dev/null
+++ b/src/master/master-client.h
@@ -0,0 +1,9 @@
+#ifndef MASTER_CLIENT_H
+#define MASTER_CLIENT_H
+
+void master_client_connected(struct service_list *service_list);
+
+void master_clients_init(void);
+void master_clients_deinit(void);
+
+#endif
diff --git a/src/master/service-listen.c b/src/master/service-listen.c
index 06da1f184ba..677b2204613 100644
--- a/src/master/service-listen.c
+++ b/src/master/service-listen.c
@@ -4,7 +4,9 @@
 #include "array.h"
 #include "fd-set-nonblock.h"
 #include "fd-close-on-exec.h"
+#include "ioloop.h"
 #include "net.h"
+#include "master-client.h"
 #ifdef HAVE_SYSTEMD
 #include "sd-daemon.h"
 #endif
@@ -337,6 +339,31 @@ static int services_verify_systemd(struct service_list *service_list)
 }
 #endif
 
+static int services_listen_master(struct service_list *service_list)
+{
+	const char *path;
+	mode_t old_umask;
+
+	path = t_strdup_printf("%s/master", service_list->set->base_dir);
+	old_umask = umask(0600 ^ 0777);
+	service_list->master_fd = net_listen_unix(path, 16);
+	if (service_list->master_fd == -1 && errno == EADDRINUSE) {
+		/* already in use. all the other sockets were fine, so just
+		   delete this and retry. */
+		i_unlink_if_exists(path);
+		service_list->master_fd = net_listen_unix(path, 16);
+	}
+	umask(old_umask);
+
+	if (service_list->master_fd == -1)
+		return 0;
+
+	service_list->io_master =
+		io_add(service_list->master_fd, IO_READ,
+		       master_client_connected, service_list);
+	return 1;
+}
+
 int services_listen(struct service_list *service_list)
 {
 	struct service *const *services;
@@ -347,6 +374,8 @@ int services_listen(struct service_list *service_list)
 		if (ret2 < ret)
 			ret = ret2;
 	}
+	if (ret > 0)
+		ret = services_listen_master(service_list);
 
 #ifdef HAVE_SYSTEMD
 	if (ret > 0)
diff --git a/src/master/service-monitor.c b/src/master/service-monitor.c
index a90c4ebe8ce..19655c2c39b 100644
--- a/src/master/service-monitor.c
+++ b/src/master/service-monitor.c
@@ -589,6 +589,11 @@ void services_monitor_stop(struct service_list *service_list, bool wait)
 		services_monitor_wait(service_list);
 	}
 
+	if (service_list->io_master != NULL)
+		io_remove(&service_list->io_master);
+	if (service_list->master_fd != -1)
+		i_close_fd(&service_list->master_fd);
+
 	array_foreach(&service_list->services, services)
 		service_monitor_stop(*services);
 
diff --git a/src/master/service.h b/src/master/service.h
index 3887e52499a..6b52a228e6f 100644
--- a/src/master/service.h
+++ b/src/master/service.h
@@ -137,6 +137,10 @@ struct service_list {
 	struct service *log;
 	struct service *anvil;
 
+	struct file_listener_settings master_listener_set;
+	struct io *io_master;
+	int master_fd;
+
 	/* nonblocking log fds usd by master */
 	int master_log_fd[2];
 	struct service_process_notify *log_byes;

From c32f2e26a0915d94ba4e9297224b7f1f508ec991 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Tue, 28 Jun 2016 23:12:03 +0300
Subject: [PATCH 425/450] doveadm: Added "service stop" command to stop
 specific services.

---
 src/doveadm/Makefile.am       |  1 +
 src/doveadm/doveadm-cmd.c     |  1 +
 src/doveadm/doveadm-cmd.h     |  1 +
 src/doveadm/doveadm-service.c | 60 +++++++++++++++++++++++++++++++++++
 4 files changed, 63 insertions(+)
 create mode 100644 src/doveadm/doveadm-service.c

diff --git a/src/doveadm/Makefile.am b/src/doveadm/Makefile.am
index eaf2726bbde..5195355d4dc 100644
--- a/src/doveadm/Makefile.am
+++ b/src/doveadm/Makefile.am
@@ -76,6 +76,7 @@ doveadm_common_cmds = \
 	doveadm-penalty.c \
 	doveadm-proxy.c \
 	doveadm-replicator.c \
+	doveadm-service.c \
 	doveadm-sis.c \
 	doveadm-stats.c \
 	doveadm-who.c
diff --git a/src/doveadm/doveadm-cmd.c b/src/doveadm/doveadm-cmd.c
index 46bb92bd6bf..3c2da477bc4 100644
--- a/src/doveadm/doveadm-cmd.c
+++ b/src/doveadm/doveadm-cmd.c
@@ -19,6 +19,7 @@ static struct doveadm_cmd *doveadm_commands[] = {
 };
 
 static struct doveadm_cmd_ver2 *doveadm_commands_ver2[] = {
+	&doveadm_cmd_service_stop_ver2,
 	&doveadm_cmd_stop_ver2,
 	&doveadm_cmd_reload_ver2,
 	&doveadm_cmd_stats_dump_ver2,
diff --git a/src/doveadm/doveadm-cmd.h b/src/doveadm/doveadm-cmd.h
index 3c677daaa7b..19a46cdedf4 100644
--- a/src/doveadm/doveadm-cmd.h
+++ b/src/doveadm/doveadm-cmd.h
@@ -146,6 +146,7 @@ bool doveadm_cmd_param_istream(const struct doveadm_cmd_context *cctx,
 void doveadm_cmd_params_clean(ARRAY_TYPE(doveadm_cmd_param_arr_t) *pargv);
 void doveadm_cmd_params_null_terminate_arrays(ARRAY_TYPE(doveadm_cmd_param_arr_t) *pargv);
 
+extern struct doveadm_cmd_ver2 doveadm_cmd_service_stop_ver2;
 extern struct doveadm_cmd_ver2 doveadm_cmd_stop_ver2;
 extern struct doveadm_cmd_ver2 doveadm_cmd_reload_ver2;
 extern struct doveadm_cmd_ver2 doveadm_cmd_stats_reset_ver2;
diff --git a/src/doveadm/doveadm-service.c b/src/doveadm/doveadm-service.c
new file mode 100644
index 00000000000..95d97188e8c
--- /dev/null
+++ b/src/doveadm/doveadm-service.c
@@ -0,0 +1,60 @@
+/* Copyright (c) 2016 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "write-full.h"
+#include "istream.h"
+#include "doveadm.h"
+
+#include <unistd.h>
+
+static void cmd_service_stop(int argc, char *argv[])
+{
+	const char *path, *line;
+	int fd;
+
+	if (argc == 1)
+		help_ver2(&doveadm_cmd_service_stop_ver2);
+
+	path = t_strconcat(doveadm_settings->base_dir, "/master", NULL);
+	fd = net_connect_unix(path);
+	if (fd == -1)
+		i_fatal("net_connect_unix(%s) failed: %m", path);
+	net_set_nonblock(fd, FALSE);
+
+	string_t *cmd = t_str_new(128);
+	str_append(cmd, "VERSION\tmaster-client\t1\t0\nSTOP");
+	for (int i = 1; i < argc; i++) {
+		str_append_c(cmd, '\t');
+		str_append(cmd, argv[i]);
+	}
+	str_append_c(cmd, '\n');
+	if (write_full(fd, str_data(cmd), str_len(cmd)) < 0)
+		i_error("write(%s) failed: %m", path);
+
+	alarm(5);
+	struct istream *input = i_stream_create_fd(fd, IO_BLOCK_SIZE, FALSE);
+	if (i_stream_read_next_line(input) == NULL ||
+	    (line = i_stream_read_next_line(input)) == NULL) {
+		i_error("read(%s) failed: %s", path, i_stream_get_error(input));
+		doveadm_exit_code = EX_TEMPFAIL;
+	} else if (line[0] == '-') {
+		doveadm_exit_code = DOVEADM_EX_NOTFOUND;
+		i_error("%s", line+1);
+	} else if (line[0] != '+') {
+		i_error("Unexpected input from %s: %s", path, line);
+		doveadm_exit_code = EX_TEMPFAIL;
+	}
+	alarm(0);
+	i_stream_destroy(&input);
+	i_close_fd(&fd);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_service_stop_ver2 = {
+	.old_cmd = cmd_service_stop,
+	.name = "service stop",
+	.usage = "<service> [<service> [...]]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "service", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};

From 923ed5836f90175e736846f02edfd9c2ee07dc6b Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 19:22:51 +0300
Subject: [PATCH 426/450] dict-sql: Treat NULL value the same as "key not
 found"

---
 src/lib-dict/dict-sql.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/lib-dict/dict-sql.c b/src/lib-dict/dict-sql.c
index feb46a67c9a..78df8e7dd11 100644
--- a/src/lib-dict/dict-sql.c
+++ b/src/lib-dict/dict-sql.c
@@ -478,6 +478,12 @@ sql_dict_lookup_async_callback(struct sql_result *sql_result,
 	else if (result.ret > 0) {
 		result.value = sql_dict_result_unescape_value(ctx->map,
 			pool_datastack_create(), sql_result);
+		if (result.value == NULL) {
+			/* NULL value returned. we'll treat this as
+			   "not found", which is probably what is usually
+			   wanted. */
+			result.ret = 0;
+		}
 	}
 	ctx->callback(&result, ctx->context);
 

From 8051312f9b78c4f47da37a7d4faef1378b8aa750 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 22:11:48 +0300
Subject: [PATCH 427/450] lib: iostream-temp: Fixed o_stream_send_istream()
 with >2GB files

---
 src/lib/iostream-temp.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib/iostream-temp.c b/src/lib/iostream-temp.c
index 907837b102d..c87ffef9fec 100644
--- a/src/lib/iostream-temp.c
+++ b/src/lib/iostream-temp.c
@@ -145,8 +145,8 @@ static int o_stream_temp_dup_cancel(struct temp_ostream *tstream)
 	return ret < 0 ? -1 : 0;
 }
 
-static int o_stream_temp_dup_istream(struct temp_ostream *outstream,
-				     struct istream *instream)
+static off_t o_stream_temp_dup_istream(struct temp_ostream *outstream,
+				       struct istream *instream)
 {
 	uoff_t in_size;
 	off_t ret;

From 47441da379ddddb16e6d68c80c30c7e70f31e07d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 20:44:37 +0300
Subject: [PATCH 428/450] doveadm user: Avoid potential crashes when running
 via doveadm-server

---
 src/doveadm/doveadm-auth-server.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/doveadm/doveadm-auth-server.c b/src/doveadm/doveadm-auth-server.c
index bb06220c3e0..eb1d9aa9fca 100644
--- a/src/doveadm/doveadm-auth-server.c
+++ b/src/doveadm/doveadm-auth-server.c
@@ -367,6 +367,7 @@ static void cmd_user_ver2(struct doveadm_cmd_context *cctx)
 	(void)doveadm_cmd_param_str(cctx, "field", &show_field);
 	(void)doveadm_cmd_param_bool(cctx, "userdb-only", &userdb_only);
 
+	memset(&input, 0, sizeof(input));
 	if (doveadm_cmd_param_array(cctx, "auth-info", &optval))
 		for(;*optval != NULL; optval++)
 			auth_user_info_parse(&input.info, *optval);

From 06aeecf47db7ebf86f5561165c904c5e8156e242 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 21:09:48 +0300
Subject: [PATCH 429/450] doveadm: Read settings with service=doveadm

This was done for mail commands while initializing the mail user, but
other commands weren't using it. This meant that doveadm was using only
global settings instead of protocol doveadm { .. } settings for everything
except mail commands.
---
 src/doveadm/doveadm.c |  1 +
 src/doveadm/main.c    | 11 +++++++++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/doveadm/doveadm.c b/src/doveadm/doveadm.c
index db15a3cc1ed..8982b9c8cec 100644
--- a/src/doveadm/doveadm.c
+++ b/src/doveadm/doveadm.c
@@ -255,6 +255,7 @@ static void doveadm_read_settings(void)
 	memset(&input, 0, sizeof(input));
 	input.roots = set_roots;
 	input.module = "doveadm";
+	input.service = "doveadm";
 	input.preserve_user = TRUE;
 	input.preserve_home = TRUE;
 	if (master_service_settings_read(master_service, &input,
diff --git a/src/doveadm/main.c b/src/doveadm/main.c
index 744d58c4cee..aca450dba56 100644
--- a/src/doveadm/main.c
+++ b/src/doveadm/main.c
@@ -101,6 +101,8 @@ int main(int argc, char *argv[])
 	};
 	enum master_service_flags service_flags =
 		MASTER_SERVICE_FLAG_KEEP_CONFIG_OPEN;
+	struct master_service_settings_input input;
+	struct master_service_settings_output output;
 	const char *error;
 	int c;
 
@@ -117,8 +119,13 @@ int main(int argc, char *argv[])
 		}
 	}
 
-	if (master_service_settings_read_simple(master_service, set_roots,
-						&error) < 0)
+	memset(&input, 0, sizeof(input));
+	input.roots = set_roots;
+	input.module = "doveadm";
+	input.service = "doveadm";
+
+	if (master_service_settings_read(master_service, &input, &output,
+					 &error) < 0)
 		i_fatal("Error reading configuration: %s", error);
 
 	master_service_init_log(master_service, "doveadm: ");

From 7be176606f0e096e13df9fa5b4ea9a161af423cf Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 22:29:20 +0300
Subject: [PATCH 430/450] dsync: When full resync is wanted in a stateful sync,
 output empty state.

This continues 3d49dc64d, which didn't actually work because
brain->require_full_resync was either cleared earlier or it was never
even set in this brain.
---
 src/doveadm/dsync/dsync-brain-mailbox.c |  4 ----
 src/doveadm/dsync/dsync-brain.c         |  9 +++++++--
 src/doveadm/dsync/dsync-ibc-pipe.c      |  9 +++++++--
 src/doveadm/dsync/dsync-ibc-private.h   |  6 ++++--
 src/doveadm/dsync/dsync-ibc-stream.c    | 13 ++++++++++---
 src/doveadm/dsync/dsync-ibc.c           | 11 +++++++----
 src/doveadm/dsync/dsync-ibc.h           |  6 ++++--
 7 files changed, 39 insertions(+), 19 deletions(-)

diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c
index fa43537bd97..bd380645bbf 100644
--- a/src/doveadm/dsync/dsync-brain-mailbox.c
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c
@@ -343,10 +343,6 @@ void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain)
 
 	i_assert(brain->box != NULL);
 
-	if (brain->require_full_resync) {
-		brain->mailbox_state.last_uidvalidity = 0;
-		brain->require_full_resync = FALSE;
-	}
 	array_append(&brain->remote_mailbox_states, &brain->mailbox_state, 1);
 	if (brain->box_exporter != NULL) {
 		const char *errstr;
diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c
index 3191708eadc..22b07aa6c93 100644
--- a/src/doveadm/dsync/dsync-brain.c
+++ b/src/doveadm/dsync/dsync-brain.c
@@ -568,16 +568,19 @@ static bool dsync_brain_finish(struct dsync_brain *brain)
 {
 	const char *error;
 	enum mail_error mail_error;
+	bool require_full_resync;
 	enum dsync_ibc_recv_ret ret;
 
 	if (!brain->master_brain) {
 		dsync_ibc_send_finish(brain->ibc,
 				      brain->failed ? "dsync failed" : NULL,
-				      brain->mail_error);
+				      brain->mail_error,
+				      brain->require_full_resync);
 		brain->state = DSYNC_STATE_DONE;
 		return TRUE;
 	} 
-	ret = dsync_ibc_recv_finish(brain->ibc, &error, &mail_error);
+	ret = dsync_ibc_recv_finish(brain->ibc, &error, &mail_error,
+				    &require_full_resync);
 	if (ret == DSYNC_IBC_RECV_RET_TRYAGAIN)
 		return FALSE;
 	if (error != NULL) {
@@ -587,6 +590,8 @@ static bool dsync_brain_finish(struct dsync_brain *brain)
 		    (brain->mail_error == 0 || brain->mail_error == MAIL_ERROR_TEMP))
 			brain->mail_error = mail_error;
 	}
+	if (require_full_resync)
+		brain->require_full_resync = TRUE;
 	brain->state = DSYNC_STATE_DONE;
 	return TRUE;
 }
diff --git a/src/doveadm/dsync/dsync-ibc-pipe.c b/src/doveadm/dsync/dsync-ibc-pipe.c
index 16b573c2a60..c8a1841d567 100644
--- a/src/doveadm/dsync/dsync-ibc-pipe.c
+++ b/src/doveadm/dsync/dsync-ibc-pipe.c
@@ -45,6 +45,7 @@ struct item {
 		struct {
 			const char *error;
 			enum mail_error mail_error;
+			bool require_full_resync;
 		} finish;
 	} u;
 };
@@ -491,7 +492,8 @@ dsync_ibc_pipe_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r)
 
 static void
 dsync_ibc_pipe_send_finish(struct dsync_ibc *ibc, const char *error,
-			   enum mail_error mail_error)
+			   enum mail_error mail_error,
+			   bool require_full_resync)
 {
 	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
 	struct item *item;
@@ -499,11 +501,13 @@ dsync_ibc_pipe_send_finish(struct dsync_ibc *ibc, const char *error,
 	item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_FINISH);
 	item->u.finish.error = p_strdup(item->pool, error);
 	item->u.finish.mail_error = mail_error;
+	item->u.finish.require_full_resync = require_full_resync;
 }
 
 static enum dsync_ibc_recv_ret
 dsync_ibc_pipe_recv_finish(struct dsync_ibc *ibc, const char **error_r,
-			   enum mail_error *mail_error_r)
+			   enum mail_error *mail_error_r,
+			   bool *require_full_resync_r)
 {
 	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
 	struct item *item;
@@ -514,6 +518,7 @@ dsync_ibc_pipe_recv_finish(struct dsync_ibc *ibc, const char **error_r,
 
 	*error_r = item->u.finish.error;
 	*mail_error_r = item->u.finish.mail_error;
+	*require_full_resync_r = item->u.finish.require_full_resync;
 	return DSYNC_IBC_RECV_RET_OK;
 }
 
diff --git a/src/doveadm/dsync/dsync-ibc-private.h b/src/doveadm/dsync/dsync-ibc-private.h
index 62112c0e9ca..9ee1d384558 100644
--- a/src/doveadm/dsync/dsync-ibc-private.h
+++ b/src/doveadm/dsync/dsync-ibc-private.h
@@ -69,10 +69,12 @@ struct dsync_ibc_vfuncs {
 			     struct dsync_mail **mail_r);
 
 	void (*send_finish)(struct dsync_ibc *ibc, const char *error,
-			    enum mail_error mail_error);
+			    enum mail_error mail_error,
+			    bool require_full_resync);
 	enum dsync_ibc_recv_ret
 		(*recv_finish)(struct dsync_ibc *ibc, const char **error_r,
-			       enum mail_error *mail_error_r);
+			       enum mail_error *mail_error_r,
+			       bool *require_full_resync_r);
 
 	void (*close_mail_streams)(struct dsync_ibc *ibc);
 	bool (*is_send_queue_full)(struct dsync_ibc *ibc);
diff --git a/src/doveadm/dsync/dsync-ibc-stream.c b/src/doveadm/dsync/dsync-ibc-stream.c
index 102fffe855c..a042fffaa0e 100644
--- a/src/doveadm/dsync/dsync-ibc-stream.c
+++ b/src/doveadm/dsync/dsync-ibc-stream.c
@@ -125,7 +125,7 @@ static const struct {
 	},
 	{ .name = "finish",
 	  .chr = 'F',
-	  .optional_keys = "error mail_error",
+	  .optional_keys = "error mail_error require_full_resync",
 	  .min_minor_version = DSYNC_PROTOCOL_MINOR_HAVE_FINISH
 	},
 	{ .name = "mailbox_cache_field",
@@ -1906,7 +1906,8 @@ dsync_ibc_stream_recv_mail(struct dsync_ibc *_ibc, struct dsync_mail **mail_r)
 
 static void
 dsync_ibc_stream_send_finish(struct dsync_ibc *_ibc, const char *error,
-			     enum mail_error mail_error)
+			     enum mail_error mail_error,
+			     bool require_full_resync)
 {
 	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
 	struct dsync_serializer_encoder *encoder;
@@ -1920,13 +1921,16 @@ dsync_ibc_stream_send_finish(struct dsync_ibc *_ibc, const char *error,
 		dsync_serializer_encode_add(encoder, "mail_error",
 					    dec2str(mail_error));
 	}
+	if (require_full_resync)
+		dsync_serializer_encode_add(encoder, "require_full_resync", "");
 	dsync_serializer_encode_finish(&encoder, str);
 	dsync_ibc_stream_send_string(ibc, str);
 }
 
 static enum dsync_ibc_recv_ret
 dsync_ibc_stream_recv_finish(struct dsync_ibc *_ibc, const char **error_r,
-			     enum mail_error *mail_error_r)
+			     enum mail_error *mail_error_r,
+			     bool *require_full_resync_r)
 {
 	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
 	struct dsync_deserializer_decoder *decoder;
@@ -1936,6 +1940,7 @@ dsync_ibc_stream_recv_finish(struct dsync_ibc *_ibc, const char **error_r,
 
 	*error_r = NULL;
 	*mail_error_r = 0;
+	*require_full_resync_r = FALSE;
 
 	p_clear(ibc->ret_pool);
 
@@ -1953,6 +1958,8 @@ dsync_ibc_stream_recv_finish(struct dsync_ibc *_ibc, const char **error_r,
 		dsync_ibc_input_error(ibc, decoder, "Invalid mail_error");
 		return DSYNC_IBC_RECV_RET_TRYAGAIN;
 	}
+	if (dsync_deserializer_decode_try(decoder, "require_full_resync", &value))
+		*require_full_resync_r = TRUE;
 	*mail_error_r = i;
 
 	ibc->finish_received = TRUE;
diff --git a/src/doveadm/dsync/dsync-ibc.c b/src/doveadm/dsync/dsync-ibc.c
index e9eea93109d..f85bed30cb7 100644
--- a/src/doveadm/dsync/dsync-ibc.c
+++ b/src/doveadm/dsync/dsync-ibc.c
@@ -196,16 +196,19 @@ dsync_ibc_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r)
 }
 
 void dsync_ibc_send_finish(struct dsync_ibc *ibc, const char *error,
-			   enum mail_error mail_error)
+			   enum mail_error mail_error,
+			   bool require_full_resync)
 {
-	ibc->v.send_finish(ibc, error, mail_error);
+	ibc->v.send_finish(ibc, error, mail_error, require_full_resync);
 }
 
 enum dsync_ibc_recv_ret
 dsync_ibc_recv_finish(struct dsync_ibc *ibc, const char **error_r,
-		      enum mail_error *mail_error_r)
+		      enum mail_error *mail_error_r,
+		      bool *require_full_resync_r)
 {
-	return ibc->v.recv_finish(ibc, error_r, mail_error_r);
+	return ibc->v.recv_finish(ibc, error_r, mail_error_r,
+				  require_full_resync_r);
 }
 
 void dsync_ibc_close_mail_streams(struct dsync_ibc *ibc)
diff --git a/src/doveadm/dsync/dsync-ibc.h b/src/doveadm/dsync/dsync-ibc.h
index d35d9211ecc..66b4026ceaf 100644
--- a/src/doveadm/dsync/dsync-ibc.h
+++ b/src/doveadm/dsync/dsync-ibc.h
@@ -148,10 +148,12 @@ enum dsync_ibc_recv_ret
 dsync_ibc_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r);
 
 void dsync_ibc_send_finish(struct dsync_ibc *ibc, const char *error,
-			   enum mail_error mail_error);
+			   enum mail_error mail_error,
+			   bool require_full_resync);
 enum dsync_ibc_recv_ret
 dsync_ibc_recv_finish(struct dsync_ibc *ibc, const char **error_r,
-		      enum mail_error *mail_error_r);
+		      enum mail_error *mail_error_r,
+		      bool *require_full_resync_r);
 
 /* Close any mail input streams that are kept open. This needs to be called
    before the mail is attempted to be freed (usually on error conditions). */

From 9892356eb820400bdfc0eb29a1be200d0a9d5e3d Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 23:54:46 +0300
Subject: [PATCH 431/450] auth: Fixed checking if delay_until is too large

---
 src/auth/auth-request.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/auth/auth-request.c b/src/auth/auth-request.c
index 4e9a428b922..072622d7c1e 100644
--- a/src/auth/auth-request.c
+++ b/src/auth/auth-request.c
@@ -1697,7 +1697,7 @@ void auth_request_set_field(struct auth_request *request,
 				"Invalid delay_until timestamp '%s'", value);
 		} else if (timestamp <= ioloop_time) {
 			/* no more delays */
-		} else if (ioloop_time - timestamp > AUTH_REQUEST_MAX_DELAY_SECS) {
+		} else if (timestamp - ioloop_time > AUTH_REQUEST_MAX_DELAY_SECS) {
 			auth_request_log_error(request, AUTH_SUBSYS_DB,
 				"delay_until timestamp %s is too much in the future, failing", value);
 			request->failed = TRUE;

From 89ed1b1592bd247f1cc54a5643f89e31ab921d30 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Wed, 29 Jun 2016 23:56:18 +0300
Subject: [PATCH 432/450] auth: delay_until can optionally now have +<max
 random secs> suffix.

---
 src/auth/auth-request.c | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/src/auth/auth-request.c b/src/auth/auth-request.c
index 072622d7c1e..3b32b405dcd 100644
--- a/src/auth/auth-request.c
+++ b/src/auth/auth-request.c
@@ -1691,10 +1691,24 @@ void auth_request_set_field(struct auth_request *request,
 		request->failed = TRUE;
 	} else if (strcmp(name, "delay_until") == 0) {
 		time_t timestamp;
+		unsigned int extra_secs = 0;
+		const char *p;
 
+		p = strchr(value, '+');
+		if (p != NULL) {
+			value = t_strdup_until(value, p++);
+			if (str_to_uint(p, &extra_secs) < 0) {
+				auth_request_log_error(request, AUTH_SUBSYS_DB,
+					"Invalid delay_until randomness number '%s'", p);
+				request->failed = TRUE;
+			} else {
+				extra_secs = rand() % extra_secs;
+			}
+		}
 		if (str_to_time(value, &timestamp) < 0) {
 			auth_request_log_error(request, AUTH_SUBSYS_DB,
 				"Invalid delay_until timestamp '%s'", value);
+			request->failed = TRUE;
 		} else if (timestamp <= ioloop_time) {
 			/* no more delays */
 		} else if (timestamp - ioloop_time > AUTH_REQUEST_MAX_DELAY_SECS) {
@@ -1702,6 +1716,10 @@ void auth_request_set_field(struct auth_request *request,
 				"delay_until timestamp %s is too much in the future, failing", value);
 			request->failed = TRUE;
 		} else {
+			/* add randomness, but not too much of it */
+			timestamp += extra_secs;
+			if (timestamp - ioloop_time > AUTH_REQUEST_MAX_DELAY_SECS)
+				timestamp = ioloop_time + AUTH_REQUEST_MAX_DELAY_SECS;
 			request->delay_until = timestamp;
 		}
 	} else if (strcmp(name, "allow_real_nets") == 0) {

From ec56d6df3f436045f0d5c36e7f7794cb1116aac8 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 30 Jun 2016 01:15:38 +0300
Subject: [PATCH 433/450] lib-ssl-iostream: Disable "CRYPTO_set_mem_functions()
 was called too late" for now

It always happens at least with Ubuntu 16.04.
---
 src/lib-ssl-iostream/dovecot-openssl-common.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-ssl-iostream/dovecot-openssl-common.c b/src/lib-ssl-iostream/dovecot-openssl-common.c
index 8e54ec74451..7d1ebf4f825 100644
--- a/src/lib-ssl-iostream/dovecot-openssl-common.c
+++ b/src/lib-ssl-iostream/dovecot-openssl-common.c
@@ -44,7 +44,7 @@ void dovecot_openssl_common_global_ref(void)
 	   conditions. */
 	if (CRYPTO_set_mem_functions(dovecot_openssl_malloc,
 				     dovecot_openssl_realloc, free) == 0)
-		i_warning("CRYPTO_set_mem_functions() was called too late");
+		/*i_warning("CRYPTO_set_mem_functions() was called too late")*/;
 
 	SSL_library_init();
 	SSL_load_error_strings();

From 2272c47816f95c2cf29b4bc8616a674d39696914 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 30 Jun 2016 01:27:37 +0300
Subject: [PATCH 434/450] lib-ssl-iostream: Compiler fix to previous commit

---
 src/lib-ssl-iostream/dovecot-openssl-common.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/lib-ssl-iostream/dovecot-openssl-common.c b/src/lib-ssl-iostream/dovecot-openssl-common.c
index 7d1ebf4f825..f4dde3233b2 100644
--- a/src/lib-ssl-iostream/dovecot-openssl-common.c
+++ b/src/lib-ssl-iostream/dovecot-openssl-common.c
@@ -43,8 +43,9 @@ void dovecot_openssl_common_global_ref(void)
 	   returning NULL. this avoids random failures on out-of-memory
 	   conditions. */
 	if (CRYPTO_set_mem_functions(dovecot_openssl_malloc,
-				     dovecot_openssl_realloc, free) == 0)
-		/*i_warning("CRYPTO_set_mem_functions() was called too late")*/;
+				     dovecot_openssl_realloc, free) == 0) {
+		/*i_warning("CRYPTO_set_mem_functions() was called too late");*/
+	}
 
 	SSL_library_init();
 	SSL_load_error_strings();

From 822ecdaeba651a25b2df926b3f52f24c6d61cfd4 Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Thu, 30 Jun 2016 09:06:33 +0300
Subject: [PATCH 435/450] dcrypt: Fix bugs in 0 and 1 byte payload files

---
 src/lib-dcrypt/istream-decrypt.c | 12 ++++++++++--
 src/lib-dcrypt/ostream-encrypt.c |  3 +++
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/lib-dcrypt/istream-decrypt.c b/src/lib-dcrypt/istream-decrypt.c
index 4ca35880b22..ae39d0d5ab0 100644
--- a/src/lib-dcrypt/istream-decrypt.c
+++ b/src/lib-dcrypt/istream-decrypt.c
@@ -60,7 +60,7 @@ ssize_t i_stream_decrypt_read_header_v1(struct decrypt_istream *stream,
 
 	const unsigned char *digest_pos = NULL, *key_digest_pos = NULL, *key_ct_pos = NULL;
 
-	size_t pos = 7;
+	size_t pos = 9;
 	size_t digest_len = 0;
 	size_t key_ct_len = 0;
 	size_t key_digest_size = 0;
@@ -585,7 +585,8 @@ ssize_t i_stream_decrypt_read_header(struct decrypt_istream *stream,
 	uint32_t hdr_len;
 	if (!get_msb32(&data, end, &hdr_len))
 		return 0;
-	if ((size_t)(end-data) < hdr_len)
+	/* do not forget stream format */
+	if ((size_t)(end-data)+1 < hdr_len)
 		return 0;
 
 	int ret;
@@ -669,6 +670,13 @@ i_stream_decrypt_read(struct istream_private *stream)
 
 		if (ret == -1 && (size == 0 || stream->parent->stream_errno != 0)) {
 			stream->istream.stream_errno = stream->parent->stream_errno;
+
+			/* file was empty */
+			if (!dstream->initialized && size == 0 && stream->parent->eof) {
+				stream->istream.eof = TRUE;
+				return -1;
+			}
+
 			if (stream->istream.stream_errno != 0)
 				return -1;
 
diff --git a/src/lib-dcrypt/ostream-encrypt.c b/src/lib-dcrypt/ostream-encrypt.c
index 0569253a0ca..d5c20a728df 100644
--- a/src/lib-dcrypt/ostream-encrypt.c
+++ b/src/lib-dcrypt/ostream-encrypt.c
@@ -496,6 +496,9 @@ int o_stream_encrypt_flush(struct ostream_private *stream)
 	const char *error;
 	struct encrypt_ostream *estream = (struct encrypt_ostream *)stream;
 
+	/* if nothing was written, we are done */
+	if (!estream->prefix_written) return o_stream_flush(stream->parent);
+
 	i_assert(!estream->finalized);
 	estream->finalized = TRUE;
 

From 9f46e5fd797dfc35856d9ae3d6fbd7adff70798b Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Thu, 30 Jun 2016 09:06:51 +0300
Subject: [PATCH 436/450] dcrypt: Add tests for 0 and 1 byte data

---
 src/lib-dcrypt/test-stream.c | 162 +++++++++++++++++++++++++++++++++--
 1 file changed, 153 insertions(+), 9 deletions(-)

diff --git a/src/lib-dcrypt/test-stream.c b/src/lib-dcrypt/test-stream.c
index 19f4b59a53b..11fd4f19b44 100644
--- a/src/lib-dcrypt/test-stream.c
+++ b/src/lib-dcrypt/test-stream.c
@@ -137,8 +137,8 @@ void test_static_v2_input(void)
 
 	while((amt = i_stream_read(is_4))>0) { i_stream_skip(is_4, amt); }
 
-        if (is_4->stream_errno != 0)
-                i_debug("error: %s", i_stream_get_error(is_4));
+	if (is_4->stream_errno != 0)
+		i_debug("error: %s", i_stream_get_error(is_4));
 
 	test_assert(is_4->stream_errno == 0);
 
@@ -187,8 +187,8 @@ void test_write_read_v1(void)
 	struct ostream *os_2 = o_stream_create_encrypt(os, "<unused>", test_v2_kp.pub, IO_STREAM_ENC_VERSION_1);
 	o_stream_nsend(os_2, payload, sizeof(payload));
 
-        if (os_2->stream_errno != 0)
-                i_debug("error: %s", o_stream_get_error(os_2));
+	if (os_2->stream_errno != 0)
+		i_debug("error: %s", o_stream_get_error(os_2));
 
 	test_assert(os_2->stream_errno == 0);
 	test_assert(o_stream_nfinish(os_2) == 0);
@@ -214,6 +214,77 @@ void test_write_read_v1(void)
 	test_end();
 }
 
+static
+void test_write_read_v1_short(void)
+{
+	test_begin("test_write_read_v1_short");
+	unsigned char payload[1];
+	const unsigned char *ptr;
+	size_t pos = 0, siz;
+	random_fill_weak(payload, 1);
+
+	struct ostream *os = iostream_temp_create("/tmp", 0);
+	struct ostream *os_2 = o_stream_create_encrypt(os, "<unused>", test_v2_kp.pub, IO_STREAM_ENC_VERSION_1);
+	o_stream_nsend(os_2, payload, sizeof(payload));
+
+	if (os_2->stream_errno != 0)
+		i_debug("error: %s", o_stream_get_error(os_2));
+
+	test_assert(os_2->stream_errno == 0);
+	test_assert(o_stream_nfinish(os_2) == 0);
+	test_assert(os_2->stream_errno == 0);
+
+	o_stream_unref(&os_2);
+
+	struct istream *is = iostream_temp_finish(&os, IO_BLOCK_SIZE);
+	struct istream *is_2 = i_stream_create_decrypt(is, test_v2_kp.priv);
+	i_stream_unref(&is);
+
+	while(i_stream_read_data(is_2, &ptr, &siz, 0)>0) {
+		test_assert_idx(pos + siz <= sizeof(payload), pos);
+		if (pos + siz > sizeof(payload)) break;
+		test_assert_idx(memcmp(ptr, payload + pos, siz) == 0, pos);
+		i_stream_skip(is_2, siz);
+	}
+
+	test_assert(is_2->stream_errno == 0);
+
+	i_stream_unref(&is_2);
+
+	test_end();
+}
+
+static
+void test_write_read_v1_empty(void)
+{
+	const unsigned char *ptr;
+	size_t siz;
+	test_begin("test_write_read_v1_empty");
+	struct ostream *os = iostream_temp_create("/tmp", 0);
+	struct ostream *os_2 = o_stream_create_encrypt(os, "<unused>", test_v1_kp.pub, IO_STREAM_ENC_VERSION_1);
+	test_assert(o_stream_nfinish(os_2) == 0);
+	if (os_2->stream_errno != 0)
+		i_debug("error: %s", o_stream_get_error(os_2));
+
+	o_stream_unref(&os_2);
+	/* this should've been enough */
+
+	struct istream *is = iostream_temp_finish(&os, IO_BLOCK_SIZE);
+	struct istream *is_2 = i_stream_create_decrypt(is, test_v1_kp.priv);
+	i_stream_unref(&is);
+
+	/* read should not fail */
+	while(i_stream_read_data(is_2, &ptr, &siz, 0)>0) {
+		test_assert(FALSE); /* should never be reached */
+	};
+
+	test_assert(is_2->stream_errno == 0);
+	if (is_2->stream_errno != 0)
+		i_debug("error: %s", i_stream_get_error(is_2));
+	i_stream_unref(&is_2);
+	test_end();
+}
+
 static
 void test_write_read_v2(void)
 {
@@ -225,10 +296,10 @@ void test_write_read_v2(void)
 
 	struct ostream *os = iostream_temp_create("/tmp", 0);
 	struct ostream *os_2 = o_stream_create_encrypt(os, "aes-256-gcm-sha256", test_v1_kp.pub, IO_STREAM_ENC_INTEGRITY_AEAD);
-	o_stream_nsend(os_2, payload, IO_BLOCK_SIZE);
+	o_stream_nsend(os_2, payload, sizeof(payload));
 	test_assert(o_stream_nfinish(os_2) == 0);
-        if (os_2->stream_errno != 0)
-                i_debug("error: %s", o_stream_get_error(os_2));
+	if (os_2->stream_errno != 0)
+		i_debug("error: %s", o_stream_get_error(os_2));
 
 	o_stream_unref(&os_2);
 
@@ -244,14 +315,83 @@ void test_write_read_v2(void)
 	}
 
 	test_assert(is_2->stream_errno == 0);
-        if (is_2->stream_errno != 0)
-                i_debug("error: %s", i_stream_get_error(is_2));
+	if (is_2->stream_errno != 0)
+		i_debug("error: %s", i_stream_get_error(is_2));
 
 	i_stream_unref(&is_2);
 
 	test_end();
 }
 
+static
+void test_write_read_v2_short(void)
+{
+	test_begin("test_write_read_v2_short");
+	unsigned char payload[1];
+	const unsigned char *ptr;
+	size_t pos = 0, siz;
+	random_fill_weak(payload, 1);
+
+	struct ostream *os = iostream_temp_create("/tmp", 0);
+	struct ostream *os_2 = o_stream_create_encrypt(os, "aes-256-gcm-sha256", test_v1_kp.pub, IO_STREAM_ENC_INTEGRITY_AEAD);
+	o_stream_nsend(os_2, payload, sizeof(payload));
+	test_assert(o_stream_nfinish(os_2) == 0);
+	if (os_2->stream_errno != 0)
+		i_debug("error: %s", o_stream_get_error(os_2));
+
+	o_stream_unref(&os_2);
+
+	struct istream *is = iostream_temp_finish(&os, IO_BLOCK_SIZE);
+	struct istream *is_2 = i_stream_create_decrypt(is, test_v1_kp.priv);
+	i_stream_unref(&is);
+
+	while(i_stream_read_data(is_2, &ptr, &siz, 0)>0) {
+		test_assert_idx(pos + siz <= sizeof(payload), pos);
+		if (pos + siz > sizeof(payload)) break;
+		test_assert_idx(memcmp(ptr, payload + pos, siz) == 0, pos);
+		i_stream_skip(is_2, siz);
+	}
+
+	test_assert(is_2->stream_errno == 0);
+	if (is_2->stream_errno != 0)
+		i_debug("error: %s", i_stream_get_error(is_2));
+
+	i_stream_unref(&is_2);
+
+	test_end();
+}
+
+static
+void test_write_read_v2_empty(void)
+{
+	const unsigned char *ptr;
+	size_t siz;
+	test_begin("test_write_read_v2_empty");
+	struct ostream *os = iostream_temp_create("/tmp", 0);
+	struct ostream *os_2 = o_stream_create_encrypt(os, "aes-256-gcm-sha256", test_v1_kp.pub, IO_STREAM_ENC_INTEGRITY_AEAD);
+	test_assert(o_stream_nfinish(os_2) == 0);
+	if (os_2->stream_errno != 0)
+		i_debug("error: %s", o_stream_get_error(os_2));
+
+	o_stream_unref(&os_2);
+	/* this should've been enough */
+
+	struct istream *is = iostream_temp_finish(&os, IO_BLOCK_SIZE);
+	struct istream *is_2 = i_stream_create_decrypt(is, test_v1_kp.priv);
+	i_stream_unref(&is);
+
+	/* read should not fail */
+	while(i_stream_read_data(is_2, &ptr, &siz, 0)>0) {
+		test_assert(FALSE); /* should never be reached */
+	};
+
+	test_assert(is_2->stream_errno == 0);
+	if (is_2->stream_errno != 0)
+		i_debug("error: %s", i_stream_get_error(is_2));
+	i_stream_unref(&is_2);
+	test_end();
+}
+
 int main(void) {
 	dcrypt_initialize("openssl", NULL, NULL);
 	random_init();
@@ -266,7 +406,11 @@ int main(void) {
 		test_static_v1_input_short,
 		test_static_v2_input,
 		test_write_read_v1,
+		test_write_read_v1_short,
+		test_write_read_v1_empty,
 		test_write_read_v2,
+		test_write_read_v2_short,
+		test_write_read_v2_empty,
 		NULL
 	};
 

From 1be434cc05d08ca680b37e3d4718d9f92d08936a Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Thu, 30 Jun 2016 09:29:26 +0300
Subject: [PATCH 437/450] dcrypt: Free keys in test-stream

---
 src/lib-dcrypt/test-stream.c | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/lib-dcrypt/test-stream.c b/src/lib-dcrypt/test-stream.c
index 11fd4f19b44..e078a8f21a0 100644
--- a/src/lib-dcrypt/test-stream.c
+++ b/src/lib-dcrypt/test-stream.c
@@ -392,6 +392,14 @@ void test_write_read_v2_empty(void)
 	test_end();
 }
 
+static
+void test_free_keys() {
+	dcrypt_key_free_private(&test_v1_kp.priv);
+	dcrypt_key_free_public(&test_v1_kp.pub);
+	dcrypt_key_free_private(&test_v2_kp.priv);
+	dcrypt_key_free_public(&test_v2_kp.pub);
+}
+
 int main(void) {
 	dcrypt_initialize("openssl", NULL, NULL);
 	random_init();
@@ -411,6 +419,7 @@ int main(void) {
 		test_write_read_v2,
 		test_write_read_v2_short,
 		test_write_read_v2_empty,
+		test_free_keys,
 		NULL
 	};
 

From 3a1d14c174e6603b19e145b624789c06d34d850a Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Thu, 30 Jun 2016 10:03:18 +0300
Subject: [PATCH 438/450] auth-request: Correctly allocate mech_password and
 credentials_scheme

---
 src/auth/auth-request.c | 48 ++++++++++++++++-------------------------
 1 file changed, 19 insertions(+), 29 deletions(-)

diff --git a/src/auth/auth-request.c b/src/auth/auth-request.c
index 3b32b405dcd..c6f278510e6 100644
--- a/src/auth/auth-request.c
+++ b/src/auth/auth-request.c
@@ -49,8 +49,6 @@ struct auth_policy_check_ctx {
 		AUTH_POLICY_CHECK_TYPE_SUCCESS,
 	} type;
 	struct auth_request *request;
-	const char *scheme;
-	char *password;
 
 	buffer_t *success_data;
 
@@ -69,11 +67,9 @@ auth_request_userdb_import(struct auth_request *request, const char *args);
 
 static
 void auth_request_verify_plain_continue(struct auth_request *request,
-					char *password,
 					verify_plain_callback_t *callback);
 static
 void auth_request_lookup_credentials_policy_continue(struct auth_request *request,
-						     const char *scheme,
 						     lookup_credentials_callback_t *callback);
 static
 void auth_request_policy_check_callback(int result, void *context);
@@ -167,7 +163,7 @@ void auth_request_success(struct auth_request *request,
 	ctx->success_data = buffer_create_dynamic(request->pool, data_size);
 	buffer_append(ctx->success_data, data, data_size);
         ctx->type = AUTH_POLICY_CHECK_TYPE_SUCCESS;
-	auth_policy_check(request, ctx->password, auth_request_policy_check_callback, ctx);
+	auth_policy_check(request, request->mech_password, auth_request_policy_check_callback, ctx);
 }
 
 static
@@ -844,10 +840,10 @@ void auth_request_policy_penalty_finish(void *context)
 
 	switch(ctx->type) {
 	case AUTH_POLICY_CHECK_TYPE_PLAIN:
-		auth_request_verify_plain_continue(ctx->request, ctx->password, ctx->callback_plain);
+		auth_request_verify_plain_continue(ctx->request, ctx->callback_plain);
 		return;
 	case AUTH_POLICY_CHECK_TYPE_LOOKUP:
-		auth_request_lookup_credentials_policy_continue(ctx->request, ctx->scheme, ctx->callback_lookup);
+		auth_request_lookup_credentials_policy_continue(ctx->request, ctx->callback_lookup);
 		return;
 	case AUTH_POLICY_CHECK_TYPE_SUCCESS:
 		auth_request_success_continue(ctx);
@@ -884,30 +880,30 @@ void auth_request_verify_plain(struct auth_request *request,
 
 	i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE);
 
+	if (request->mech_password == NULL)
+		request->mech_password = p_strdup(request->pool, password);
+	else
+		i_assert(request->mech_password == password);
+
 	if (request->policy_processed) {
-		auth_request_verify_plain_continue(request,
-			p_strdup(request->pool, password), callback);
+		auth_request_verify_plain_continue(request, callback);
 	} else {
 		ctx = p_new(request->pool, struct auth_policy_check_ctx, 1);
 		ctx->request = request;
-		if (request->mech_password == NULL)
-			ctx->password = p_strdup(request->pool, password);
-		else
-			ctx->password = request->mech_password;
 		ctx->callback_plain = callback;
 		ctx->type = AUTH_POLICY_CHECK_TYPE_PLAIN;
-		auth_policy_check(request, ctx->password, auth_request_policy_check_callback, ctx);
+		auth_policy_check(request, request->mech_password, auth_request_policy_check_callback, ctx);
 	}
 }
 
 static
 void auth_request_verify_plain_continue(struct auth_request *request,
-					char *password,
 					verify_plain_callback_t *callback) {
 
 	struct auth_passdb *passdb;
 	enum passdb_result result;
 	const char *cache_key;
+	const char *password = request->mech_password;
 
 	i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE);
 
@@ -924,11 +920,7 @@ void auth_request_verify_plain_continue(struct auth_request *request,
 	}
 
         passdb = request->passdb;
-	if (request->mech_password == NULL)
-		/* this is allocated on start */
-		request->mech_password = password;
-	else
-		i_assert(request->mech_password == password);
+
 	request->private_callback.verify_plain = callback;
 
 	cache_key = passdb_cache == NULL ? NULL : passdb->cache_key;
@@ -1051,24 +1043,24 @@ void auth_request_lookup_credentials(struct auth_request *request,
 
 	i_assert(request->state == AUTH_REQUEST_STATE_MECH_CONTINUE);
 
+	if (request->credentials_scheme == NULL)
+		request->credentials_scheme = p_strdup(request->pool, scheme);
+	else
+		i_assert(request->credentials_scheme == scheme);
+
 	if (request->policy_processed)
-		auth_request_lookup_credentials_policy_continue(request, scheme, callback);
+		auth_request_lookup_credentials_policy_continue(request, callback);
 	else {
 		ctx = p_new(request->pool, struct auth_policy_check_ctx, 1);
 		ctx->request = request;
-		if (request->credentials_scheme == NULL)
-			ctx->scheme = p_strdup(request->pool, scheme);
-		else
-			ctx->scheme = request->credentials_scheme;
 		ctx->callback_lookup = callback;
 		ctx->type = AUTH_POLICY_CHECK_TYPE_LOOKUP;
-		auth_policy_check(request, ctx->password, auth_request_policy_check_callback, ctx);
+		auth_policy_check(request, ctx->request->mech_password, auth_request_policy_check_callback, ctx);
 	}
 }
 
 static
 void auth_request_lookup_credentials_policy_continue(struct auth_request *request,
-						     const char *scheme,
 						     lookup_credentials_callback_t *callback)
 {
 	struct auth_passdb *passdb;
@@ -1083,8 +1075,6 @@ void auth_request_lookup_credentials_policy_continue(struct auth_request *reques
 	}
 	passdb = request->passdb;
 
-	if (request->credentials_scheme == NULL)
-		request->credentials_scheme = p_strdup(request->pool, scheme);
 	request->private_callback.lookup_credentials = callback;
 
 	cache_key = passdb_cache == NULL ? NULL : passdb->cache_key;

From 4fd02abe13ade0f8e136a0f31378523634ee3f3f Mon Sep 17 00:00:00 2001
From: Aki Tuomi <aki.tuomi@dovecot.fi>
Date: Thu, 30 Jun 2016 11:57:19 +0300
Subject: [PATCH 439/450] auth-request: Whitespace fix

---
 src/auth/auth-request.c | 90 ++++++++++++++++++++---------------------
 1 file changed, 45 insertions(+), 45 deletions(-)

diff --git a/src/auth/auth-request.c b/src/auth/auth-request.c
index c6f278510e6..89ec55d869b 100644
--- a/src/auth/auth-request.c
+++ b/src/auth/auth-request.c
@@ -162,7 +162,7 @@ void auth_request_success(struct auth_request *request,
 	ctx->request = request;
 	ctx->success_data = buffer_create_dynamic(request->pool, data_size);
 	buffer_append(ctx->success_data, data, data_size);
-        ctx->type = AUTH_POLICY_CHECK_TYPE_SUCCESS;
+	 ctx->type = AUTH_POLICY_CHECK_TYPE_SUCCESS;
 	auth_policy_check(request, request->mech_password, auth_request_policy_check_callback, ctx);
 }
 
@@ -283,7 +283,7 @@ void auth_request_export(struct auth_request *request, string_t *dest)
 
 	auth_str_add_keyvalue(dest, "service", request->service);
 
-        if (request->master_user != NULL) {
+	 if (request->master_user != NULL) {
 		auth_str_add_keyvalue(dest, "master-user",
 				      request->master_user);
 	}
@@ -517,9 +517,9 @@ static void auth_request_save_cache(struct auth_request *request,
 		   strdup() it so that mech_password doesn't get
 		   cleared too early. */
 		if (!password_generate_encoded(request->mech_password,
-					       request->user,
-					       CACHED_PASSWORD_SCHEME,
-					       &encoded_password))
+						request->user,
+						CACHED_PASSWORD_SCHEME,
+						&encoded_password))
 			i_unreached();
 		request->passdb_password =
 			p_strconcat(request->pool, "{"CACHED_PASSWORD_SCHEME"}",
@@ -643,7 +643,7 @@ auth_request_handle_passdb_callback(enum passdb_result *result,
 		return TRUE;
 	case PASSDB_RESULT_PASS_EXPIRED:
 		auth_request_set_field(request, "reason",
-				       "Password expired", NULL);
+					"Password expired", NULL);
 		return TRUE;
 
 	case PASSDB_RESULT_OK:
@@ -703,7 +703,7 @@ auth_request_handle_passdb_callback(enum passdb_result *result,
 		next_passdb = request->passdb->next;
 	}
 	while (next_passdb != NULL &&
-	       auth_request_want_skip_passdb(request, next_passdb))
+		auth_request_want_skip_passdb(request, next_passdb))
 		next_passdb = next_passdb->next;
 
 	if (*result == PASSDB_RESULT_OK) {
@@ -725,7 +725,7 @@ auth_request_handle_passdb_callback(enum passdb_result *result,
 
 	if (passdb_continue && next_passdb != NULL) {
 		/* try next passdb. */
-                request->passdb = next_passdb;
+		  request->passdb = next_passdb;
 		request->passdb_password = NULL;
 
 		if (*result == PASSDB_RESULT_USER_UNKNOWN) {
@@ -873,8 +873,8 @@ void auth_request_policy_check_callback(int result, void *context)
 }
 
 void auth_request_verify_plain(struct auth_request *request,
-			       const char *password,
-			       verify_plain_callback_t *callback)
+				const char *password,
+				verify_plain_callback_t *callback)
 {
 	struct auth_policy_check_ctx *ctx;
 
@@ -919,7 +919,7 @@ void auth_request_verify_plain_continue(struct auth_request *request,
 		return;
 	}
 
-        passdb = request->passdb;
+	 passdb = request->passdb;
 
 	request->private_callback.verify_plain = callback;
 
@@ -949,9 +949,9 @@ void auth_request_verify_plain_continue(struct auth_request *request,
 
 static void
 auth_request_lookup_credentials_finish(enum passdb_result result,
-				       const unsigned char *credentials,
-				       size_t size,
-				       struct auth_request *request)
+					const unsigned char *credentials,
+					size_t size,
+					struct auth_request *request)
 {
 	passdb_template_export(request->passdb->override_fields_tmpl, request);
 	if (!auth_request_handle_passdb_callback(&result, request)) {
@@ -969,7 +969,7 @@ auth_request_lookup_credentials_finish(enum passdb_result result,
 		}
 		auth_request_lookup_credentials(request,
 			request->credentials_scheme,
-                	request->private_callback.lookup_credentials);
+		  	request->private_callback.lookup_credentials);
 	} else {
 		if (request->delayed_credentials != NULL && size == 0) {
 			/* we did multiple passdb lookups, but the last one
@@ -1032,7 +1032,7 @@ void auth_request_lookup_credentials_callback(enum passdb_result result,
 	}
 
 	auth_request_lookup_credentials_finish(result, credentials, size,
-					       request);
+						request);
 }
 
 void auth_request_lookup_credentials(struct auth_request *request,
@@ -1184,13 +1184,13 @@ static bool auth_request_lookup_user_cache(struct auth_request *request,
 	if (value == NULL || (expired && !use_expired)) {
 		stats->auth_cache_miss_count++;
 		auth_request_log_debug(request, AUTH_SUBSYS_DB,
-				       value == NULL ? "userdb cache miss" :
-				       "userdb cache expired");
+					value == NULL ? "userdb cache miss" :
+					"userdb cache expired");
 		return FALSE;
 	}
 	stats->auth_cache_hit_count++;
 	auth_request_log_debug(request, AUTH_SUBSYS_DB,
-			       "userdb cache hit: %s", value);
+				"userdb cache hit: %s", value);
 
 	if (*value == '\0') {
 		/* negative cache entry */
@@ -1257,7 +1257,7 @@ void auth_request_userdb_callback(enum userdb_result result,
 
 	next_userdb = userdb->next;
 	while (next_userdb != NULL &&
-	       auth_request_want_skip_userdb(request, next_userdb))
+		auth_request_want_skip_userdb(request, next_userdb))
 		next_userdb = next_userdb->next;
 
 	if (userdb_continue && next_userdb != NULL) {
@@ -1322,7 +1322,7 @@ void auth_request_userdb_callback(enum userdb_result result,
 		}
 	}
 
-        request->private_callback.userdb(result, request);
+	 request->private_callback.userdb(result, request);
 }
 
 void auth_request_lookup_user(struct auth_request *request,
@@ -1366,7 +1366,7 @@ void auth_request_lookup_user(struct auth_request *request,
 
 static char *
 auth_request_fix_username(struct auth_request *request, const char *username,
-                          const char **error_r)
+			     const char **error_r)
 {
 	const struct auth_settings *set = request->set;
 	unsigned char *p;
@@ -1375,12 +1375,12 @@ auth_request_fix_username(struct auth_request *request, const char *username,
 	if (*set->default_realm != '\0' &&
 	    strchr(username, '@') == NULL) {
 		user = p_strconcat(request->pool, username, "@",
-                                   set->default_realm, NULL);
+					set->default_realm, NULL);
 	} else {
 		user = p_strdup(request->pool, username);
 	}
 
-        for (p = (unsigned char *)user; *p != '\0'; p++) {
+	 for (p = (unsigned char *)user; *p != '\0'; p++) {
 		if (set->username_translation_map[*p & 0xff] != 0)
 			*p = set->username_translation_map[*p & 0xff];
 		if (set->username_chars_map[*p & 0xff] == 0) {
@@ -1414,11 +1414,11 @@ auth_request_fix_username(struct auth_request *request, const char *username,
 		*error_r = "Empty username";
 		return NULL;
 	}
-        return user;
+	 return user;
 }
 
 bool auth_request_set_username(struct auth_request *request,
-			       const char *username, const char **error_r)
+				const char *username, const char **error_r)
 {
 	const struct auth_settings *set = request->set;
 	const char *p, *login_username = NULL;
@@ -1447,7 +1447,7 @@ bool auth_request_set_username(struct auth_request *request,
 		username = request->user;
 	}
 
-        request->user = auth_request_fix_username(request, username, error_r);
+	 request->user = auth_request_fix_username(request, username, error_r);
 	if (request->user == NULL)
 		return FALSE;
 	if (request->translated_username == NULL) {
@@ -1465,8 +1465,8 @@ bool auth_request_set_username(struct auth_request *request,
 }
 
 bool auth_request_set_login_username(struct auth_request *request,
-                                     const char *username,
-                                     const char **error_r)
+					  const char *username,
+					  const char **error_r)
 {
 	struct auth_passdb *master_passdb;
 
@@ -1481,7 +1481,7 @@ bool auth_request_set_login_username(struct auth_request *request,
 		return TRUE;
 	}
 
-        /* lookup request->user from masterdb first */
+	 /* lookup request->user from masterdb first */
 	master_passdb = auth_request_get_auth(request)->masterdbs;
 	if (master_passdb == NULL) {
 		*error_r = "Master user login attempted without master passdbs";
@@ -1489,21 +1489,21 @@ bool auth_request_set_login_username(struct auth_request *request,
 	}
 	request->passdb = master_passdb;
 
-        request->requested_login_user =
-                auth_request_fix_username(request, username, error_r);
+	request->requested_login_user =
+		auth_request_fix_username(request, username, error_r);
 	if (request->requested_login_user == NULL)
 		return FALSE;
 
 	auth_request_log_debug(request, AUTH_SUBSYS_DB,
-			       "Master user lookup for login: %s",
-			       request->requested_login_user);
+				"Master user lookup for login: %s",
+				request->requested_login_user);
 	return TRUE;
 }
 
 static void
 auth_request_validate_networks(struct auth_request *request,
-			       const char *name, const char *networks,
-			       const struct ip_addr *remote_ip)
+				const char *name, const char *networks,
+				const struct ip_addr *remote_ip)
 {
 	const char *const *net;
 	struct ip_addr net_ip;
@@ -1618,8 +1618,8 @@ auth_request_try_update_username(struct auth_request *request,
 
 	if (strcmp(request->user, new_value) != 0) {
 		auth_request_log_debug(request, AUTH_SUBSYS_DB,
-				       "username changed %s -> %s",
-				       request->user, new_value);
+					"username changed %s -> %s",
+					request->user, new_value);
 		request->user = p_strdup(request->pool, new_value);
 	}
 	return TRUE;
@@ -1814,7 +1814,7 @@ static void auth_request_set_uidgid_file(struct auth_request *request,
 	auth_request_var_expand(path, path_template, request, NULL);
 	if (stat(str_c(path), &st) < 0) {
 		auth_request_log_error(request, AUTH_SUBSYS_DB,
-				       "stat(%s) failed: %m", str_c(path));
+					"stat(%s) failed: %m", str_c(path));
 	} else {
 		auth_fields_add(request->userdb_reply,
 				"uid", dec2str(st.st_uid), 0);
@@ -1987,7 +1987,7 @@ auth_request_proxy_finish_ip(struct auth_request *request,
 		/* proxying to ourself - log in without proxying by dropping
 		   all the proxying fields. */
 		bool proxy_always = auth_fields_exists(request->extra_fields,
-						       "proxy_always");
+							"proxy_always");
 
 		auth_request_proxy_finish_failure(request);
 		if (proxy_always) {
@@ -2074,7 +2074,7 @@ static int auth_request_proxy_host_lookup(struct auth_request *request,
 	request->dns_lookup_ctx = ctx;
 
 	if (dns_lookup(host, &dns_set, auth_request_proxy_dns_callback, ctx,
-		       &ctx->dns_lookup) < 0) {
+			&ctx->dns_lookup) < 0) {
 		/* failed early */
 		return -1;
 	}
@@ -2256,7 +2256,7 @@ int auth_request_password_verify(struct auth_request *request,
 
 	if (auth_fields_exists(request->extra_fields, "nopassword")) {
 		auth_request_log_debug(request, subsystem,
-				       "Allowing any password");
+					"Allowing any password");
 		return 1;
 	}
 
@@ -2269,7 +2269,7 @@ int auth_request_password_verify(struct auth_request *request,
 				scheme, error);
 		} else {
 			auth_request_log_error(request, subsystem,
-					       "Unknown scheme %s", scheme);
+						"Unknown scheme %s", scheme);
 		}
 		return -1;
 	}
@@ -2283,8 +2283,8 @@ int auth_request_password_verify(struct auth_request *request,
 		const char *password_str = request->set->debug_passwords ?
 			t_strdup_printf(" '%s'", crypted_password) : "";
 		auth_request_log_error(request, subsystem,
-				       "Invalid password%s in passdb: %s",
-				       password_str, error);
+					"Invalid password%s in passdb: %s",
+					password_str, error);
 	} else if (ret == 0) {
 		auth_request_log_password_mismatch(request, subsystem);
 	}

From 129abc3e5df9411d20f5c53b22f81376b917e093 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 30 Jun 2016 12:08:19 +0300
Subject: [PATCH 440/450] lmtp: Fixed pipelining of commands after RCPT TO

We should stop while waiting for anvil reply. Broken by ced943b0a.
---
 src/lmtp/commands.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lmtp/commands.c b/src/lmtp/commands.c
index b65c1bf7869..fcc3633b8b1 100644
--- a/src/lmtp/commands.c
+++ b/src/lmtp/commands.c
@@ -746,7 +746,7 @@ int cmd_rcpt(struct client *client, const char *args)
 					rcpt_anvil_lookup_callback, rcpt);
 		/* stop processing further commands while anvil query is
 		   pending */
-		return rcpt->anvil_query != NULL ? 0 : -1;
+		return rcpt->anvil_query == NULL ? 0 : -1;
 	}
 }
 

From dbca72da2fb4596cf731d3c9f47ced530c165f4b Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 30 Jun 2016 12:10:29 +0300
Subject: [PATCH 441/450] lmtp: Removed assert, which triggers if anvil connect
 fails.

---
 src/lmtp/commands.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/lmtp/commands.c b/src/lmtp/commands.c
index fcc3633b8b1..480c28dc43d 100644
--- a/src/lmtp/commands.c
+++ b/src/lmtp/commands.c
@@ -602,8 +602,6 @@ static void rcpt_anvil_lookup_callback(const char *reply, void *context)
 	const struct mail_storage_service_input *input;
 	unsigned int parallel_count = 0;
 
-	i_assert(rcpt->anvil_query != NULL);
-
 	rcpt->anvil_query = NULL;
 	if (reply == NULL) {
 		/* lookup failed */

From 1969dd8f3de622465680c491709a66e8fa4e2823 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 30 Jun 2016 14:28:55 +0300
Subject: [PATCH 442/450] Released v2.2.25.rc1.

---
 NEWS         | 136 +++++++++++++++++++++++++++++++++++++++++++++++++++
 configure.ac |   4 +-
 2 files changed, 138 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 52315d15681..4f72c0ba2fe 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,139 @@
+v2.2.25 2016-06-xx  Timo Sirainen <tss@iki.fi>
+
+	* lmtp: Start tracking lmtp_user_concurrency_limit and reject already
+	  at RCPT TO stage. This avoids MTA unnecessarily completing DATA only
+	  to get an error.
+	* doveadm: Previously only mail settings were read from protocol
+	  doveadm { .. } section. Now all settings are.
+
+	+ quota: Added quota_over_flag_lazy_check setting. It avoids checking
+	  quota_over_flag always at startup. Instead it's checked only when
+	  quota is being read for some other purpose.
+	+ auth: Added a new auth policy service:
+	  http://wiki2.dovecot.org/Authentication/Policy
+	+ auth: Added PBKDF2 password scheme
+	+ auth: Added %{auth_user}, %{auth_username} and %{auth_domain}
+	+ auth: Added ":remove" suffix to extra field names to remove them.
+	+ auth: Added "delay_until=<timestamp>[+<max random secs>]" passdb
+	  extra field. The auth will wait until <timestamp> and optionally some
+	  randomness and then return success.
+	+ dict proxy: Added idle_msecs=<n> parameter. Support async operations.
+	+ Performance improvements for handling large mailboxes.
+	+ Added lib-dcrypt API for providing cryptographic functions.
+	+ Added "doveadm mailbox update" command
+	+ imap commands' output now includes timing spent on the "syncing"
+	  stage if it's larger than 0.
+	+ cassandra: Added metrics=<path> to connect setting to output internal
+	  statistics in JSON format every second to <path>.
+	+ doveadm mailbox delete: Added -e parameter to delete only empty
+	  mailboxes. Added --unsafe option to quickly delete a mailbox,
+	  bypassing lazy_expunge and quota plugins.
+	+ doveadm user & auth cache flush are now available via doveadm-server.
+	+ doveadm service stop <services> will stop specified services while
+	  leaving the rest of Dovecot running.
+	+ quota optimization: Avoid reading mail sizes for backends which
+	  don't need them (count, fs, dirsize)
+	+ Added mailbox { autoexpunge_max_mails=<n> } setting.
+	+ Added welcome plugin: http://wiki2.dovecot.org/Plugins/Welcome
+	+ fts: Added fts_autoindex_exclude setting.
+	- v2.2.24's MIME parser was assert-crashing on mails having truncated
+	  MIME headers.
+	- auth: With multiple userdbs the final success/failure result wasn't
+	  always correct. The last userdb's result was always used.
+	- doveadm backup was sometimes deleting entire mailboxes unnecessarily.
+	- doveadm: Command -parameters weren't being sent to doveadm-server.
+	- If dovecot.index read failed e.g. because mmap() reached VSZ limit,
+	  an empty index could have been opened instead, corrupting the
+	  mailbox state.
+	- imapc: Fixed EXPUNGE handling when imapc_features didn't have modseq.
+	- lazy-expunge: Fixed a crash when copying failed. Various other fixes.
+	- fts-lucene: Fixed crash on index rescan.
+	- auth_stats=yes produced broken output
+	- dict-ldap: Various fixes
+	- dict-sql: NULL values crashed. Now they're treated as "not found".
+
+v2.2.24 2016-04-26  Timo Sirainen <tss@iki.fi>
+
+	* doveconf now warns if it sees a global setting being changed when
+	  the same setting was already set inside some filters. (A common
+	  mistake has been adding more plugins to a global mail_plugins
+	  setting after it was already set inside protocol { .. }, which
+	  caused the global setting to be ignored for that protocol.)
+	* LMTP proxy: Increased default timeout 30s -> 125s. This makes it
+	  less likely to reach the timeout and cause duplicate deliveries.
+	* LMTP and indexer now append ":suffix" to session IDs to make it
+	  unique for the specific user's delivery. (Fixes duplicate session
+	  ID warnings in stats process.)
+
+	+ Added dict-ldap for performing read-only LDAP dict lookups.
+	+ lazy-expunge: All mails can be saved to a single specified mailbox.
+	+ mailbox { autoexpunge } supports now wildcards in mailbox names.
+	+ doveadm HTTP API: Added support for proxy commands
+	+ imapc: Reconnect when getting disconnected in non-selected state.
+	+ imapc: Added imapc_features=modseq to access MODSEQs/HIGHESTMODSEQ.
+	  This is especially useful for incremental dsync.
+	+ doveadm auth/user: Auth lookup performs debug logging if
+	  -o auth_debug=yes is given to doveadm.
+	+ Added passdb/userdb { auth_verbose=yes|no } setting.
+	+ Cassandra: Added user, password, num_threads, connect_timeout and
+	  request_timeout settings.
+	+ doveadm user -e <value>: Print <value> with %variables expanded.
+	- Huge header lines could have caused Dovecot to use too much memory
+	  (depending on config and used IMAP commands). (Typically this would
+	  result in only the single user's process dying with out of memory
+	  due to reaching service { vsz_limit } - not a global DoS).
+	- dsync: Detect and handle invalid/stale -s state string better.
+	- dsync: Fixed crash caused by specific mailbox renames
+	- auth: Auth cache is now disabled passwd-file. It was unnecessary and
+	  it broke %variables in extra fields.
+	- fts-tika: Don't crash if it returns 500 error
+	- dict-redis: Fixed timeout handling
+	- SEARCH INTHREAD was crashing
+	- stats: Only a single fifo_listeners was supported, making it
+	  impossible to use both auth_stats=yes and mail stats plugin.
+	- SSL errors were logged in separate "Stacked error" log lines
+	  instead of as part of the disconnection reason.
+	- MIME body parser didn't handle properly when a child MIME part's
+	  --boundary had the same prefix as the parent.
+
+v2.2.23 2016-03-30  Timo Sirainen <tss@iki.fi>
+
+	- Various fixes to doveadm. Especially running commands via
+	  doveadm-server was broken.
+	- director: Fixed user weakness getting stuck in some situations
+	- director: Fixed a situation where directors keep re-sending
+	  different states to each others and never becoming synced.
+	- director: Fixed assert-crash related to a slow "user killed" reply
+	- Fixed assert-crash related to istream-concat, which could have
+	  been triggered at least by a Sieve script.
+
+v2.2.22 2016-03-16  Timo Sirainen <tss@iki.fi>
+
+	+ Added doveadm HTTP API: See
+	  http://wiki2.dovecot.org/Design/DoveadmProtocol/HTTP
+	+ virtual plugin: Mailbox filtering can now be done based on the
+	  mailbox metadata. See http://wiki2.dovecot.org/Plugins/Virtual
+	+ stats: Added doveadm stats reset to reset global stats.
+	+ stats: Added authentication statistics if auth_stats=yes.
+	+ dsync, imapc, pop3c & pop3-migration: Many optimizations,
+	  improvements and error handling fixes.
+	+ doveadm: Most commands now stop soon after SIGINT/SIGTERM.
+	- auth: Auth caching was done too aggressively when %variables were
+	  used in default_fields, override_fields or LDAP pass/user_attrs.
+	  userdb result_* were also ignored when user was found from cache.
+	- imap: Fixed various assert-crashes caused v2.2.20+. Some of them
+	  caught actual hangs or otherwise unwanted behavior towards IMAP
+	  clients.
+	- Expunges were forgotten in some situations, for example when
+	  pipelining multiple IMAP MOVE commands.
+	- quota: Per-namespaces quota were broken for dict and count backends
+	  in v2.2.20+
+	- fts-solr: Search queries were using OR instead of AND as the
+	  separator for multi-token search queries in v2.2.20+.
+	- Single instance storage support wasn't really working in v2.2.16+
+	- dbox: POP3 message ordering wasn't working correctly.
+	- virtual plugin: Fixed crashes related to backend mailbox deletions.
+
 v2.2.21 2015-12-11  Timo Sirainen <tss@iki.fi>
 
 	- doveadm mailbox list (and some others) were broken in v2.2.20
diff --git a/configure.ac b/configure.ac
index e3283d0ee39..28831ef2b44 100644
--- a/configure.ac
+++ b/configure.ac
@@ -2,8 +2,8 @@ AC_PREREQ([2.59])
 
 # Be sure to update ABI version also if anything changes that might require
 # recompiling plugins. Most importantly that means if any structs are changed.
-AC_INIT([Dovecot],[2.2.devel],[dovecot@dovecot.org])
-AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv20($PACKAGE_VERSION)", [Dovecot ABI version])
+AC_INIT([Dovecot],[2.2.25.rc1],[dovecot@dovecot.org])
+AC_DEFINE_UNQUOTED([DOVECOT_ABI_VERSION], "2.2.ABIv25($PACKAGE_VERSION)", [Dovecot ABI version])
 
 AC_CONFIG_SRCDIR([src])
 

From 3c432ac593a4df5658eec0e88e03285a1345a75a Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 30 Jun 2016 14:08:24 +0300
Subject: [PATCH 443/450] master: Stopping didn't close dead-pipes early
 enough.

This caused stopping to be too slow. Broken by 0153cf542.
---
 src/master/service-monitor.c | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/master/service-monitor.c b/src/master/service-monitor.c
index 19655c2c39b..44cdb2696b2 100644
--- a/src/master/service-monitor.c
+++ b/src/master/service-monitor.c
@@ -505,6 +505,14 @@ void services_monitor_start(struct service_list *service_list)
 	}
 }
 
+static void service_monitor_close_dead_pipe(struct service *service)
+{
+	if (service->master_dead_pipe_fd[0] != -1) {
+		i_close_fd(&service->master_dead_pipe_fd[0]);
+		i_close_fd(&service->master_dead_pipe_fd[1]);
+	}
+}
+
 void service_monitor_stop(struct service *service)
 {
 	int i;
@@ -522,10 +530,7 @@ void service_monitor_stop(struct service *service)
 			service->status_fd[i] = -1;
 		}
 	}
-	if (service->master_dead_pipe_fd[0] != -1) {
-		i_close_fd(&service->master_dead_pipe_fd[0]);
-		i_close_fd(&service->master_dead_pipe_fd[1]);
-	}
+	service_monitor_close_dead_pipe(service);
 	if (service->login_notify_fd != -1) {
 		if (close(service->login_notify_fd) < 0) {
 			service_error(service,
@@ -582,6 +587,9 @@ void services_monitor_stop(struct service_list *service_list, bool wait)
 {
 	struct service *const *services;
 
+	array_foreach(&service_list->services, services)
+		service_monitor_close_dead_pipe(*services);
+
 	if (wait) {
 		/* we've notified all children that the master is dead.
 		   now wait for the children to either die or to tell that

From abd85e981f5380f16fa641a7e3d6444dbb83ab08 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 30 Jun 2016 16:21:40 +0300
Subject: [PATCH 444/450] imap: Fixed reading LDA settings from config.

---
 src/imap/imap-settings.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/imap/imap-settings.c b/src/imap/imap-settings.c
index d1668e6910a..d3e869cf806 100644
--- a/src/imap/imap-settings.c
+++ b/src/imap/imap-settings.c
@@ -5,6 +5,7 @@
 #include "settings-parser.h"
 #include "service-settings.h"
 #include "mail-storage-settings.h"
+#include "lda-settings.h"
 #include "imap-settings.h"
 
 #include <stddef.h>
@@ -101,6 +102,7 @@ static const struct imap_settings imap_default_settings = {
 
 static const struct setting_parser_info *imap_setting_dependencies[] = {
 	&mail_user_setting_parser_info,
+	&lda_setting_parser_info,
 	NULL
 };
 

From 88a6c0df00e0fd4421001e65048f0604eeb91fbe Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Thu, 30 Jun 2016 18:13:52 +0300
Subject: [PATCH 445/450] lib-dcrypt: Compiling fix

---
 src/lib-dcrypt/dcrypt.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-dcrypt/dcrypt.c b/src/lib-dcrypt/dcrypt.c
index 7592aa8a91f..2269923c61b 100644
--- a/src/lib-dcrypt/dcrypt.c
+++ b/src/lib-dcrypt/dcrypt.c
@@ -136,7 +136,7 @@ bool dcrypt_ctx_sym_final(struct dcrypt_context_symmetric *ctx, buffer_t *result
 
 void dcrypt_ctx_sym_set_padding(struct dcrypt_context_symmetric *ctx, bool padding)
 {
-	return dcrypt_vfs->ctx_sym_set_padding(ctx, padding);
+	dcrypt_vfs->ctx_sym_set_padding(ctx, padding);
 }
 
 bool dcrypt_ctx_hmac_create(const char *algorithm, struct dcrypt_context_hmac **ctx_r, const char **error_r)

From 975f69e14154b4238a280f27cca863ad60818d7c Mon Sep 17 00:00:00 2001
From: Stephan Bosch <stephan@rename-it.nl>
Date: Thu, 30 Jun 2016 22:35:59 +0200
Subject: [PATCH 446/450] lib-http: client: Fixed potential segfault problem in
 http_client_connection_server_close().

Used wrong variable: it is set to NULL by http_client_request_unref() before it is used.
Problem found by Coverity.
---
 src/lib-http/http-client-connection.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib-http/http-client-connection.c b/src/lib-http/http-client-connection.c
index 66adf75040d..567a0df3756 100644
--- a/src/lib-http/http-client-connection.c
+++ b/src/lib-http/http-client-connection.c
@@ -113,7 +113,7 @@ http_client_connection_server_close(struct http_client_connection **_conn)
 		if (!http_client_request_unref(req_idx))
 			continue;
 		/* resubmit the request, which may drop it */
-		if ((*req_idx)->state < HTTP_REQUEST_STATE_FINISHED)
+		if (req->state < HTTP_REQUEST_STATE_FINISHED)
 			http_client_request_resubmit(req);
 	}	
 	array_clear(&conn->request_wait_list);

From 522bb22d0ba9bad5e1d10c8363340e7ad9874d75 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 1 Jul 2016 01:11:02 +0300
Subject: [PATCH 447/450] configure: Check if C99 "static" keyword is supported
 for array sizes.

---
 configure.ac     | 18 ++++++++++++++++++
 src/lib/macros.h |  3 +--
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/configure.ac b/configure.ac
index 28831ef2b44..2ebc28e5ad0 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1007,6 +1007,24 @@ if test $i_cv_signed_time_t = yes; then
   AC_DEFINE(TIME_T_SIGNED,, [Define if your time_t is signed])
 fi
 
+AC_CACHE_CHECK([if we can use C99 static in array sizes],i_cv_c99_static_arrays,[
+  AC_TRY_COMPILE([
+    void foo(int arr[static 20]);
+  ], [
+  ], [
+    i_cv_c99_static_arrays=yes
+  ], [
+    i_cv_c99_static_arrays=no
+  ])
+])
+
+if test $i_cv_c99_static_arrays = yes; then
+  static_value=static
+else
+  static_value=
+fi
+AC_DEFINE_UNQUOTED(STATIC_ARRAY, $static_value, [C99 static array])
+
 dnl Our implementation of AC_C_FLEXIBLE_ARRAY_MEMBER.
 dnl Use it until autoconf 2.61+ becomes more widely used
 AC_CACHE_CHECK([if we can use C99-like flexible array members],i_cv_c99_flex_arrays,[
diff --git a/src/lib/macros.h b/src/lib/macros.h
index 2faf0c8cf17..593ca49d44c 100644
--- a/src/lib/macros.h
+++ b/src/lib/macros.h
@@ -227,9 +227,8 @@
 #endif
 
 #ifdef __cplusplus
+#  undef STATIC_ARRAY
 #  define STATIC_ARRAY
-#else
-#  define STATIC_ARRAY static
 #endif
 
 #endif

From ad7302c393e346df09c8a11074d518f9b2d8f7a3 Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 1 Jul 2016 02:49:32 +0300
Subject: [PATCH 448/450] login-proxy: cork+uncork in flush callbacks

Potentially reduces latency at the end of a larger data transfer.
---
 src/login-common/login-proxy.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/login-common/login-proxy.c b/src/login-common/login-proxy.c
index 85e40df03e8..e24d8eb032b 100644
--- a/src/login-common/login-proxy.c
+++ b/src/login-common/login-proxy.c
@@ -197,10 +197,12 @@ static void proxy_client_disconnected_input(struct login_proxy *proxy)
 static int server_output(struct login_proxy *proxy)
 {
 	proxy->last_io = ioloop_time;
+	o_stream_cork(proxy->server_output);
 	if (o_stream_flush(proxy->server_output) < 0) {
 		login_proxy_free_ostream(&proxy, proxy->server_output, TRUE);
 		return 1;
 	}
+	o_stream_uncork(proxy->server_output);
 
 	if (proxy->client_io == NULL &&
 	    o_stream_get_buffer_used_size(proxy->server_output) <
@@ -216,10 +218,12 @@ static int server_output(struct login_proxy *proxy)
 static int proxy_client_output(struct login_proxy *proxy)
 {
 	proxy->last_io = ioloop_time;
+	o_stream_cork(proxy->client_output);
 	if (o_stream_flush(proxy->client_output) < 0) {
 		login_proxy_free_ostream(&proxy, proxy->client_output, FALSE);
 		return 1;
 	}
+	o_stream_uncork(proxy->client_output);
 
 	if (proxy->server_io == NULL &&
 	    o_stream_get_buffer_used_size(proxy->client_output) <

From 8d3f4a62f28c3973c05e6749bed53efe0693f51e Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 1 Jul 2016 11:07:47 +0300
Subject: [PATCH 449/450] doveadm mailbox update: Avoid assert-crash on errors.

---
 src/doveadm/doveadm-mail-mailbox.c | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/doveadm/doveadm-mail-mailbox.c b/src/doveadm/doveadm-mail-mailbox.c
index 411f37556df..a300e844b65 100644
--- a/src/doveadm/doveadm-mail-mailbox.c
+++ b/src/doveadm/doveadm-mail-mailbox.c
@@ -639,6 +639,7 @@ int cmd_mailbox_update_run(struct doveadm_mail_cmd_context *_ctx,
 	struct update_cmd_context *ctx = (struct update_cmd_context *)_ctx;
 	struct mail_namespace *ns;
 	struct mailbox *box;
+	enum mail_error mail_error;
 	int ret = 0;
 
 	ns = mail_namespace_find(user->namespaces, ctx->mailbox);
@@ -647,7 +648,8 @@ int cmd_mailbox_update_run(struct doveadm_mail_cmd_context *_ctx,
 	if ((ret = mailbox_update(box, &(ctx->update))) != 0) {
 		i_error("Cannot update %s: %s",
 			ctx->mailbox,
-			mailbox_get_last_error(box, NULL));
+			mailbox_get_last_error(box, &mail_error));
+		doveadm_mail_failed_error(_ctx, mail_error);
 	}
 
 	mailbox_free(&box);

From b2f9b1d4f89c7c241ac12e526b3331f3a371677e Mon Sep 17 00:00:00 2001
From: Timo Sirainen <timo.sirainen@dovecot.fi>
Date: Fri, 1 Jul 2016 11:12:03 +0300
Subject: [PATCH 450/450] lib-storage: Autoexpunging should ignore nonexistent
 mailboxes.

The check was done too late after recent changes.
---
 src/lib-storage/mail-autoexpunge.c | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/lib-storage/mail-autoexpunge.c b/src/lib-storage/mail-autoexpunge.c
index a9d0054043d..9956f866b6c 100644
--- a/src/lib-storage/mail-autoexpunge.c
+++ b/src/lib-storage/mail-autoexpunge.c
@@ -28,8 +28,13 @@ mailbox_autoexpunge(struct mailbox *box, unsigned int interval_time,
 
 	/* first try to check quickly from mailbox list index if we should
 	   bother opening this mailbox. */
-	if (mailbox_get_status(box, STATUS_MESSAGES, &status) < 0)
+	if (mailbox_get_status(box, STATUS_MESSAGES, &status) < 0) {
+		if (mailbox_get_last_mail_error(box) == MAIL_ERROR_NOTFOUND) {
+			/* autocreated mailbox doesn't exist yet */
+			return 0;
+		}
 		return -1;
+	}
 	if (interval_time == 0 && status.messages <= max_mails)
 		return 0;
 
@@ -42,13 +47,8 @@ mailbox_autoexpunge(struct mailbox *box, unsigned int interval_time,
 			return 0;
 	}
 
-	if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST) < 0) {
-		if (mailbox_get_last_mail_error(box) == MAIL_ERROR_NOTFOUND) {
-			/* autocreated mailbox doesn't exist yet */
-			return 0;
-		}
+	if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FAST) < 0)
 		return -1;
-	}
 
 	t = mailbox_transaction_begin(box, 0);
 	mail = mail_alloc(t, 0, NULL);