Enable payload compression (#201)

* enable HTTP payload compression with zlib

This commit adds the option to compress the HTTP/REST payload
(CBOR/JSON):
- zlib is added as a build dependency, libcurl is now compiled against
zlib and zlib is statically linked into the driver DLL;
- a new connection string "Compress" controls the addition of the
Accept-Encoding HTTP header, with possible values "on", "off" and "auto"
(which adds the header if encryption is not enabled);
- the possibility to build and ship libcurl as a dynamic library along
the driver has been removed.

Additionally, this commit switches the default payload format to CBOR.

* add zlib license files

add zlib license files

* fix outdated comment

Update comment about installing zlib files.

* fix another c&p wrong comment

Correct code comment (copy&paste error).

Author: bpintea

CMakeLists.txt

@@ -193,37 +193,46 @@ set(CTIMESTAMP_PATH_SRC ${CMAKE_SOURCE_DIR}/libs/c-timestamp CACHE PATH
 aux_source_directory(${CTIMESTAMP_PATH_SRC}/ DRV_SRC)
 
 #
-# add libcurl to the project
+# add libcurl (and zlib) to the project
 #
 set(LIBCURL_PATH_SRC ${CMAKE_SOURCE_DIR}/libs/curl CACHE PATH
 	"Lib curl source path")
-set(LIBCURL_LINK_MODE static CACHE STRING
-	"Lib curl linking mode: static (default) or dll")
 set(LIBCURL_BUILD_TYPE debug CACHE STRING
 	"Lib curl build type: debug (default) or release")
 
+# zlib paths
+set(ZLIB_PATH_SRC ${CMAKE_SOURCE_DIR}/libs/zlib CACHE PATH
+	"Lib zlib source path")
+set(ZLIB_PATH_INST ${CMAKE_BINARY_DIR}/zlib CACHE PATH
+	"Lib zlib install path")
+
 if (${LIBCURL_BUILD_TYPE} MATCHES [Dd][Ee][Bb][Uu][Gg])
 	set(LIBCURL_DEBUG_ENABLED yes)
 	set(LIBCURL_BUILD_TYPE debug)
 	set(LIBCURL_BUILD_SUFFIX _debug)
+	# zlib's nmake-based "win32" build system contains a default,
+	# non-modifiable "-MD". Setting ZLIB_LOC will override it, which will
+	# issue a compiler warning (that can be ignored, can't be supressed).
+	set(ZLIB_LOC -MDd)
 else (${LIBCURL_BUILD_TYPE} MATCHES [Dd][Ee][Bb][Uu][Gg])
 	set(LIBCURL_DEBUG_ENABLED no)
 	set(LIBCURL_BUILD_TYPE release)
 	# empty LIBCURL_BUILD_SUFFIX
+	# empty ZLIB_LOC
 endif (${LIBCURL_BUILD_TYPE} MATCHES [Dd][Ee][Bb][Uu][Gg])
 
 set(LIBCURL_LD_PATH
 	# Curl "installs" the .dll and .lib in different directories -> use the
 	# build dir to find both files in same directory instead of installing.
 	# Curl's win build root directory is not configurable.
-	# The path built below is only constant for the subtree tag (current:
-	# 7.61.0) and default/below nmake options (only with: IPv6, SSPI, WinSSL).
-	${LIBCURL_PATH_SRC}/builds/libcurl-vc-${TARCH}-${LIBCURL_BUILD_TYPE}-${LIBCURL_LINK_MODE}-ipv6-sspi-winssl-obj-lib/
+	# The path built below is only constant for the subtree tag and the nmake
+	# options below: ZLIB, IPv6, SSPI, WinSSL, plus IDN, not echoed in name.
+	${LIBCURL_PATH_SRC}/builds/libcurl-vc-${TARCH}-${LIBCURL_BUILD_TYPE}-static-zlib-static-ipv6-sspi-winssl-obj-lib/
 	CACHE PATH "Lib curl load library path")
 set(LIBCURL_INC_PATH ${LIBCURL_PATH_SRC}/include CACHE PATH
 	"Lib curl include path")
 
-# Build libcurl.
+# Build zlib, then libcurl.
 # Note: this happens at config time as a pre-requisite, for now. This might
 # be changed to a build target later (possibly as a CMake subproject: re-link
 # only if out-of-date, skip building the .exe, allow disabling non-HTTP
@@ -232,14 +241,38 @@ set(LIBCURL_INC_PATH ${LIBCURL_PATH_SRC}/include CACHE PATH
 # entire build "single-config", since the build type (rel/dbg) is decided at
 # CMake-generation, not along the MSBuild invocation.
 if (NOT IS_DIRECTORY ${LIBCURL_LD_PATH})
+	# build zlib first.
+	message("Building zlib library in ${ZLIB_PATH_SRC}")
+	execute_process(COMMAND
+		# zlib's "win32" makefile builds in situ => always clean before
+		# building, potentially removing builds of different architecture than
+		# current's build.
+		nmake /f win32/Makefile.msc clean zlib.lib LOC=${ZLIB_LOC}
+		RESULT_VARIABLE CMD_RETURN
+		WORKING_DIRECTORY "${ZLIB_PATH_SRC}"
+		)
+	if (${CMD_RETURN})
+		message(FATAL_ERROR "Building zlib failed.")
+	endif (${CMD_RETURN})
+	# libcurl expects a /lib and a /include folder under the location provided
+	# as the path to zlib. zlib/win32's makefile has no install target, so
+	# we'll just cmake-install them under the building dir.
+	file(INSTALL ${ZLIB_PATH_SRC}/zlib.lib DESTINATION ${ZLIB_PATH_INST}/lib)
+	file(GLOB ZLIB_H_FILES LIST_DIRECTORIES false ${ZLIB_PATH_SRC}
+		${ZLIB_PATH_SRC}/*.h)
+	file(INSTALL ${ZLIB_H_FILES} DESTINATION ${ZLIB_PATH_INST}/include)
+
+	# build libcurl second: config first, build afterwards..
+	message("Building curl library in ${LIBCURL_PATH_SRC}")
 	execute_process(COMMAND buildconf.bat
 		RESULT_VARIABLE CMD_RETURN
 		WORKING_DIRECTORY "${LIBCURL_PATH_SRC}"
 		)
 	if (NOT ${CMD_RETURN})
 		execute_process(COMMAND
-			nmake /f Makefile.vc mode=${LIBCURL_LINK_MODE} MACHINE=${TARCH}
+			nmake /f Makefile.vc mode=static MACHINE=${TARCH}
 			ENABLE_WINSSL=yes ENABLE_IDN=yes ENABLE_IPV6=yes ENABLE_SSPI=yes
+			WITH_ZLIB=static ZLIB_PATH=${ZLIB_PATH_INST}
 			# build type needs to be synchronized (to link in the same CRT)
 			DEBUG=${LIBCURL_DEBUG_ENABLED}
 			# This "sneaks in" a define to disable all other protocols than
@@ -260,32 +293,31 @@ endif(NOT IS_DIRECTORY ${LIBCURL_LD_PATH})
 
 # add libcurl as dependency
 if (${WIN32})
-	if (${LIBCURL_LINK_MODE} MATCHES [Dd][Ll][Ll])
-		add_library(libcurl SHARED IMPORTED)
-		set_property(TARGET libcurl PROPERTY IMPORTED_LOCATION
-			${LIBCURL_LD_PATH}/libcurl${LIBCURL_BUILD_SUFFIX}${CMAKE_SHARED_LIBRARY_SUFFIX})
-		set_property(TARGET libcurl PROPERTY IMPORTED_IMPLIB
-			${LIBCURL_LD_PATH}/libcurl${LIBCURL_BUILD_SUFFIX}${CMAKE_IMPORT_LIBRARY_SUFFIX})
-		# empty LIBCURL_WIN_LIBS
-	else (${LIBCURL_LINK_MODE} MATCHES [Dd][Ll][Ll])
-		add_library(libcurl STATIC IMPORTED)
-		set_property(TARGET libcurl PROPERTY IMPORTED_LOCATION
-			${LIBCURL_LD_PATH}/libcurl_a${LIBCURL_BUILD_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
-		set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /DCURL_STATICLIB")
-		# Libraries that libcurl/WinSSL links against.
-		# Removed: wldap32 advapi32 gdi32 user32 (unused with current config)
-		set(LIBCURL_WIN_LIBS ws2_32 crypt32 normaliz)
-	endif (${LIBCURL_LINK_MODE} MATCHES [Dd][Ll][Ll])
+	add_library(zlib STATIC IMPORTED)
+	set_property(TARGET zlib PROPERTY IMPORTED_LOCATION
+		${ZLIB_PATH_INST}/lib/zlib.lib)
+
+	add_library(libcurl STATIC IMPORTED)
+	set_property(TARGET libcurl PROPERTY IMPORTED_LOCATION
+		${LIBCURL_LD_PATH}/libcurl_a${LIBCURL_BUILD_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX})
+	set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /DCURL_STATICLIB")
+	# Libraries that libcurl/WinSSL links against.
+	# Removed: wldap32 advapi32 gdi32 user32 (unused with current config)
+	set(LIBCURL_WIN_LIBS ws2_32 crypt32 normaliz)
 else (${WIN32})
 	set_property(TARGET libcurl PROPERTY IMPORTED_LOCATION
 		${LIBCURL_LD_PATH}/libcurl${CMAKE_SHARED_LIBRARY_SUFFIX})
 endif (${WIN32})
 
 add_custom_target(curlclean
-	COMMAND nmake /f Makefile.vc mode=${LIBCURL_LINK_MODE} clean
+	COMMAND nmake /f Makefile.vc mode=static clean
 	COMMAND ../buildconf.bat -clean
 		WORKING_DIRECTORY "${LIBCURL_PATH_SRC}/winbuild"
 	)
+add_custom_target(zlibclean
+	COMMAND nmake /f win32/Makefile.msc clean
+		WORKING_DIRECTORY "${ZLIB_PATH_SRC}"
+	)
 
 #
 # add tinycbor to the project
@@ -376,7 +408,7 @@ include_directories(${ODBC_INC} ${DRV_SRC_DIR} ${LIBCURL_INC_PATH}
 	${DSNEDITOR_INC_PATH})
 target_link_libraries(${DRV_NAME} odbccp32 legacy_stdio_definitions
 	${DSNBND_LIB_BIN_DIR_BASE}-$<CONFIG>/esdsnbnd${BARCH}${CMAKE_IMPORT_LIBRARY_SUFFIX}
-	libcurl ${LIBCURL_WIN_LIBS})
+	zlib libcurl ${LIBCURL_WIN_LIBS})
 
 
 #
@@ -405,12 +437,6 @@ install(FILES
 	LICENSE.rtf LICENSE.txt ${CMAKE_BINARY_DIR}/NOTICE.txt
 	DESTINATION ${INSTALL_DIR})
 # add libcurl if build dynamically
-if (${LIBCURL_LINK_MODE} MATCHES [Dd][Ll][Ll])
-install(FILES
-	# need to use FILE : https://public.kitware.com/Bug/view.php?id=14311
-	${LIBCURL_LD_PATH}/libcurl${LIBCURL_BUILD_SUFFIX}${CMAKE_SHARED_LIBRARY_SUFFIX}
-	DESTINATION ${INSTALL_DIR})
-endif (${LIBCURL_LINK_MODE} MATCHES [Dd][Ll][Ll])
 # add editor DLLs
 install(FILES
 	${DSNBND_LIB_BIN_DIR_BASE}-$<CONFIG>/esdsnedt${CMAKE_SHARED_LIBRARY_SUFFIX}

README.md

@@ -61,6 +61,7 @@ The required libraries are added as subtrees to the project, in the libs directo
       |_libs
         |_ODBC-Specification
         |_curl
+        |_zlib
         |_c-timestamp
         |_ujson4c
         |_tinycbor

build.bat

@@ -297,7 +297,6 @@ REM USAGE function: output a usage message
 	echo Extra development arguments:
 	echo    nobuild     : skip project building (the default is to build^).
 	echo    genonly     : generate project/make files, but don't build.
-	echo    curldll     : link libcurl dynamically.
 	echo    exports     : dump the exported symbols in the DLL after building.
 	echo    depends     : dump the dependents libs of the build DLL.
 	echo    install[:D] : install the driver files. D, the target directory
@@ -335,6 +334,9 @@ REM CTESTS function: run CI testing
 REM PROPER function: clean up the build and libs dir.
 :PROPER
 	echo %~nx0: cleaning libs.
+	if exist %BUILD_DIR%\zlibclean.vcxproj (
+		MSBuild %BUILD_DIR%\zlibclean.vcxproj
+	)
 	if exist %BUILD_DIR%\curlclean.vcxproj (
 		MSBuild %BUILD_DIR%\curlclean.vcxproj
 	)
@@ -502,9 +504,6 @@ REM BUILD function: build various targets
 		REM no explicit x86 generator and is the default (MSVC2017 only?).
 		set CMAKE_ARGS=!CMAKE_ARGS! -DCMAKE_GENERATOR_PLATFORM=%TARCH:x86=%
 
-		if /i not [%ARG:curldll=%] == [%ARG%] (
-			set CMAKE_ARGS=!CMAKE_ARGS! -DLIBCURL_LINK_MODE=dll
-		)
 		if /i [!BUILD_TYPE!] == [Debug] (
 			set CMAKE_ARGS=!CMAKE_ARGS! -DLIBCURL_BUILD_TYPE=debug
 		) else (
@@ -621,7 +620,7 @@ REM TESTS_SUITE_S function: run the compiled unit tests
 
 	goto:eof
 
-REM INSTALL_DO function: copy DLLs (libcurl, odbc) into the install
+REM INSTALL_DO function: copy DLLs into the install
 :INSTALL_DO
 	echo %~nx0: installing the driver files.
 	MSBuild INSTALL.vcxproj !MSBUILD_ARGS!

devtools/3rd_party/licenses/zlib-INFO.csv

@@ -0,0 +1,2 @@
+name,version,revision,url,license,copyright
+zlib,1.2.11,,https://www.zlib.net,zlib License,"Copyright (C) 1995-2016 Jean-loup Gailly, Mark Adler"

devtools/3rd_party/licenses/zlib-LICENSE.txt

@@ -0,0 +1,21 @@
+Copyright (C) 1995-2017 Jean-loup Gailly and Mark Adler
+
+This software is provided 'as-is', without any express or implied
+warranty.  In no event will the authors be held liable for any damages
+arising from the use of this software.
+
+Permission is granted to anyone to use this software for any purpose,
+including commercial applications, and to alter it and redistribute it
+freely, subject to the following restrictions:
+
+1. The origin of this software must not be misrepresented; you must not
+   claim that you wrote the original software. If you use this software
+   in a product, an acknowledgment in the product documentation would be
+   appreciated but is not required.
+2. Altered source versions must be plainly marked as such, and must not be
+   misrepresented as being the original software.
+3. This notice may not be removed or altered from any source distribution.
+
+Jean-loup Gailly        Mark Adler
+[email protected]          [email protected]
+

devtools/3rd_party/licenses/zlib-NOTICE.txt

@@ -437,6 +437,7 @@ static SQLRETURN dbc_curl_init(esodbc_dbc_st *dbc)
 {
 	CURL *curl;
 	SQLRETURN ret;
+	BOOL compress;
 
 	assert(! dbc->curl);
 
@@ -473,6 +474,16 @@ static SQLRETURN dbc_curl_init(esodbc_dbc_st *dbc)
 		goto err;
 	}
 
+	/* set the accepted encoding (compression) header */
+	compress = dbc->compression == ESODBC_CMPSS_ON ||
+		(dbc->compression == ESODBC_CMPSS_AUTO && !dbc->secure);
+	dbc->curl_err = curl_easy_setopt(curl, CURLOPT_ACCEPT_ENCODING,
+			compress ? "" : NULL);
+	if (dbc->curl_err != CURLE_OK) {
+		ERRH(dbc, "libcurl: failed to set HTTP headers list.");
+		goto err;
+	}
+
 	if (dbc->secure) {
 		dbc->curl_err = curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER,
 				ESODBC_SEC_CHECK_CA <= dbc->secure ? 1L : 0L);
@@ -513,6 +524,13 @@ static SQLRETURN dbc_curl_init(esodbc_dbc_st *dbc)
 		 * (CURLOPT_SSL_FALSESTART), CURLOPT_SSL_VERIFYSTATUS,
 		 * CURLOPT_PROXY_*
 		 */
+
+		/* TLS has its own compression options (RFC3749), so doing it twice
+		 * will likelyl be detrimental, but there's also the security
+		 * implication (CVE-2012-4929). */
+		if (compress) {
+			WARNH(dbc, "compression and encryption are both enabled.");
+		}
 	}
 
 	/* set authentication parameters */
@@ -1361,6 +1379,18 @@ SQLRETURN config_dbc(esodbc_dbc_st *dbc, esodbc_dsn_attrs_st *attrs)
 	}
 	INFOH(dbc, "pack JSON: %s.", dbc->pack_json ? "true" : "false");
 
+	/*
+	 * set the compression option: auto/on/off
+	 */
+	if (EQ_CASE_WSTR(&attrs->compression, &MK_WSTR(ESODBC_DSN_CMPSS_AUTO))) {
+		dbc->compression = ESODBC_CMPSS_AUTO;
+	} else {
+		dbc->compression = wstr2bool(&attrs->compression) ?
+			ESODBC_CMPSS_ON : ESODBC_CMPSS_OFF;
+	}
+	INFOH(dbc, "compression: %d (" LWPDL ").", dbc->compression,
+		LWSTR(&attrs->compression));
+
 	/* "apply TZ" param for time conversions */
 	dbc->apply_tz = wstr2bool(&attrs->apply_tz);
 	INFOH(dbc, "apply TZ: %s.", dbc->apply_tz ? "true" : "false");

driver/connect.c

@@ -164,7 +164,9 @@
 /* don't follow redirection from the server  */
 #define ESODBC_DEF_FOLLOW			"yes"
 /* packing of REST bodies (JSON or CBOR) */
-#define ESODBC_DEF_PACKING			ESODBC_DSN_PACK_JSON
+#define ESODBC_DEF_PACKING			ESODBC_DSN_PACK_CBOR
+/* zlib compression of REST bodies (auto/true/false) */
+#define ESODBC_DEF_COMPRESSION		ESODBC_DSN_CMPSS_AUTO
 /* default tracing activation */
 #define ESODBC_DEF_TRACE_ENABLED	"0"
 /* default tracing level */

driver/defs.h

@@ -72,6 +72,7 @@ int assign_dsn_attr(esodbc_dsn_attrs_st *attrs,
 		{&MK_WSTR(ESODBC_DSN_FOLLOW), &attrs->follow},
 		{&MK_WSTR(ESODBC_DSN_CATALOG), &attrs->catalog},
 		{&MK_WSTR(ESODBC_DSN_PACKING), &attrs->packing},
+		{&MK_WSTR(ESODBC_DSN_COMPRESSION), &attrs->compression},
 		{&MK_WSTR(ESODBC_DSN_MAX_FETCH_SIZE), &attrs->max_fetch_size},
 		{&MK_WSTR(ESODBC_DSN_MAX_BODY_SIZE_MB), &attrs->max_body_size},
 		{&MK_WSTR(ESODBC_DSN_APPLY_TZ), &attrs->apply_tz},
@@ -406,6 +407,7 @@ long TEST_API write_00_list(esodbc_dsn_attrs_st *attrs,
 		{&MK_WSTR(ESODBC_DSN_FOLLOW), &attrs->follow},
 		{&MK_WSTR(ESODBC_DSN_CATALOG), &attrs->catalog},
 		{&MK_WSTR(ESODBC_DSN_PACKING), &attrs->packing},
+		{&MK_WSTR(ESODBC_DSN_COMPRESSION), &attrs->compression},
 		{&MK_WSTR(ESODBC_DSN_MAX_FETCH_SIZE), &attrs->max_fetch_size},
 		{&MK_WSTR(ESODBC_DSN_MAX_BODY_SIZE_MB), &attrs->max_body_size},
 		{&MK_WSTR(ESODBC_DSN_APPLY_TZ), &attrs->apply_tz},
@@ -657,6 +659,10 @@ BOOL write_system_dsn(esodbc_dsn_attrs_st *new_attrs,
 			&MK_WSTR(ESODBC_DSN_PACKING), &new_attrs->packing,
 			old_attrs ? &old_attrs->packing : NULL
 		},
+		{
+			&MK_WSTR(ESODBC_DSN_COMPRESSION), &new_attrs->compression,
+			old_attrs ? &old_attrs->packing : NULL
+		},
 		{
 			&MK_WSTR(ESODBC_DSN_MAX_FETCH_SIZE), &new_attrs->max_fetch_size,
 			old_attrs ? &old_attrs->max_fetch_size : NULL
@@ -776,6 +782,7 @@ long TEST_API write_connection_string(esodbc_dsn_attrs_st *attrs,
 		{&attrs->follow, &MK_WSTR(ESODBC_DSN_FOLLOW)},
 		{&attrs->catalog, &MK_WSTR(ESODBC_DSN_CATALOG)},
 		{&attrs->packing, &MK_WSTR(ESODBC_DSN_PACKING)},
+		{&attrs->compression, &MK_WSTR(ESODBC_DSN_COMPRESSION)},
 		{&attrs->max_fetch_size, &MK_WSTR(ESODBC_DSN_MAX_FETCH_SIZE)},
 		{&attrs->max_body_size, &MK_WSTR(ESODBC_DSN_MAX_BODY_SIZE_MB)},
 		{&attrs->apply_tz, &MK_WSTR(ESODBC_DSN_APPLY_TZ)},
@@ -863,6 +870,9 @@ void assign_dsn_defaults(esodbc_dsn_attrs_st *attrs)
 	res |= assign_dsn_attr(attrs,
 			&MK_WSTR(ESODBC_DSN_PACKING), &MK_WSTR(ESODBC_DEF_PACKING),
 			/*overwrite?*/FALSE);
+	res |= assign_dsn_attr(attrs,
+			&MK_WSTR(ESODBC_DSN_COMPRESSION), &MK_WSTR(ESODBC_DEF_COMPRESSION),
+			/*overwrite?*/FALSE);
 	res |= assign_dsn_attr(attrs,
 			&MK_WSTR(ESODBC_DSN_MAX_FETCH_SIZE),
 			&MK_WSTR(ESODBC_DEF_FETCH_SIZE),

driver/dsn.c

@@ -31,6 +31,7 @@
 #define ESODBC_DSN_FOLLOW			"Follow"
 #define ESODBC_DSN_CATALOG			"Catalog"
 #define ESODBC_DSN_PACKING			"Packing"
+#define ESODBC_DSN_COMPRESSION		"Compression"
 #define ESODBC_DSN_MAX_FETCH_SIZE	"MaxFetchSize"
 #define ESODBC_DSN_MAX_BODY_SIZE_MB	"MaxBodySizeMB"
 #define ESODBC_DSN_APPLY_TZ			"ApplyTZ"
@@ -46,6 +47,10 @@
 /* Packing values */
 #define ESODBC_DSN_PACK_JSON		"JSON"
 #define ESODBC_DSN_PACK_CBOR		"CBOR"
+/* Compression values */
+#define ESODBC_DSN_CMPSS_AUTO		"auto"
+#define ESODBC_DSN_CMPSS_ON			"on"
+#define ESODBC_DSN_CMPSS_OFF		"off"
 /* VersionChecking values */
 #define ESODBC_DSN_VC_STRICT		"strict"
 #define ESODBC_DSN_VC_MAJOR			"major"
@@ -73,6 +78,7 @@ typedef struct {
 	wstr_st follow;
 	wstr_st catalog;
 	wstr_st packing;
+	wstr_st compression;
 	wstr_st max_fetch_size;
 	wstr_st max_body_size;
 	wstr_st apply_tz;
@@ -84,7 +90,7 @@ typedef struct {
 	wstr_st trace_enabled;
 	wstr_st trace_file;
 	wstr_st trace_level;
-#define ESODBC_DSN_ATTRS_COUNT	27
+#define ESODBC_DSN_ATTRS_COUNT	28
 	SQLWCHAR buff[ESODBC_DSN_ATTRS_COUNT * ESODBC_DSN_MAX_ATTR_LEN];
 	/* DSN reading/writing functions are passed a SQLSMALLINT lenght param */
 #if SHRT_MAX < ESODBC_DSN_ATTRS_COUNT * ESODBC_DSN_MAX_ATTR_LEN