[WIP] Update libzip to 1.8.0

Enable zstd support

TODO: hide all the zlib and BZIP2 symbols

Author: grendello

.gitmodules

@@ -21,3 +21,6 @@
 	path = external/zlib
 	url = https://github.com/madler/zlib.git
 	branch = master
+[submodule "external/zstd"]
+	path = external/zstd
+	url = https://github.com/facebook/zstd.git

CMakeLists.txt

@@ -46,11 +46,21 @@ set(ENABLE_GNUTLS OFF CACHE BOOL "Do not use gnutls for libzip" FORCE)
 set(ENABLE_OPENSSL OFF CACHE BOOL "Do not use OpenSSL for libzip" FORCE)
 set(ENABLE_MBEDTLS OFF CACHE BOOL "Do not use mbedtls for libzip" FORCE)
 set(ENABLE_WINDOWS_CRYPTO OFF CACHE BOOL "Do not use Windows Crypto" FORCE)
+set(ENABLE_ZSTD ON CACHE BOOL "Use zstd in libzip" FORCE)
 
 if(NOT ENABLE_XZ)
   set(ENABLE_LZMA OFF CACHE BOOL "Do not use XZ for libzip" FORCE)
 endif()
 
+#
+# zstd
+#
+set(ZSTD_MULTITHREAD_SUPPORT OFF CACHE BOOL "Do not build zstd with multi-thread support" FORCE)
+set(ZSTD_BUILD_STATIC ON CACHE BOOL "Build static zstd" FORCE)
+set(ZSTD_BUILD_SHARED OFF CACHE BOOL "Do not build shared zstd" FORCE)
+set(ZSTD_BUILD_PROGRAMS OFF CACHE BOOL "Do not build zstd programs" FORCE)
+set(ZSTD_BUILD_TESTS OFF CACHE BOOL "Do not build zstd tests" FORCE)
+
 #
 # zlib-ng
 #
@@ -298,11 +308,20 @@ if(BUILD_DEPENDENCIES)
     add_subdirectory(external/zlib)
   endif()
 
+  if(UNIX)
+    set(ZLIB_EXTERN "-DZEXTERN=__attribute__((visibility(\"hidden\")))")
+    target_compile_options(
+      zlib
+      PRIVATE
+      -fvisibility=hidden
+      )
+  endif()
+
   target_compile_options(
     zlib
     PRIVATE
     ${LZS_C_FLAGS}
-    -fvisibility=hidden
+    "${ZLIB_EXTERN}"
     )
 
 if(NOT WIN32 AND ENABLE_XZ)
@@ -316,6 +335,23 @@ if(NOT WIN32 AND ENABLE_XZ)
     )
 endif()
 
+  add_subdirectory(external/zstd/build/cmake)
+  if(UNIX)
+    set(ZSTD_EXTERN "-DZSTDLIB_VISIBILITY=__attribute__((visibility(\"hidden\")))")
+    target_compile_options(
+      libzstd_static
+      PRIVATE
+      -fvisibility=hidden
+      )
+  endif()
+
+  target_compile_options(
+    libzstd_static
+    PRIVATE
+    ${LZS_C_FLAGS}
+    "${ZSTD_EXTERN}"
+    )
+
     set(BZIP2_SOURCES external/bzip2)
     add_library(
       bz2
@@ -329,11 +365,18 @@ endif()
       ${BZIP2_SOURCES}/bzlib.c
       )
 
+    if(UNIX)
+      target_compile_options(
+        bz2
+        PRIVATE
+        ${LZS_C_FLAGS}
+        )
+    endif()
+
     target_compile_options(
       bz2
       PRIVATE
       ${LZS_C_FLAGS}
-      -fvisibility=hidden
       )
 
     configure_file("${CMAKE_SOURCE_DIR}/data/bzip2.pc.in" "${CMAKE_BINARY_DIR}/bzip2.pc")
@@ -371,6 +414,11 @@ else()
 
   list(PREPEND CMAKE_PREFIX_PATH "${ARTIFACTS_ROOT_DIR}")
 
+  if(WIN32)
+    # If we don't rename the library, it won't be detected by libzip's cmake
+    file(RENAME ${ARTIFACTS_ROOT_DIR}/lib/zstd_static.lib ${ARTIFACTS_ROOT_DIR}/lib/zstd.lib)
+  endif()
+
   add_subdirectory(external/libzip)
 
   target_compile_options(
@@ -381,6 +429,8 @@ else()
 
   if(UNIX)
     set(ZIP_EXTERN "-DZIP_EXTERN=__attribute__((visibility(\"default\")))")
+    set(ZSTD_EXTERN "-DZSTDLIB_VISIBILITY=__attribute__((visibility(\"hidden\")))")
+    set(ZLIB_EXTERN "-DZEXTERN=__attribute__((visibility(\"hidden\")))")
     target_compile_options(
       zip
       PRIVATE
@@ -393,6 +443,8 @@ else()
     zip
     PRIVATE
     "${ZIP_EXTERN}"
+    "${ZSTD_EXTERN}"
+    "${ZLIB_EXTERN}"
     )
 
   if(APPLE)
@@ -422,6 +474,7 @@ else()
   else()
     message(STATUS "LZMA: DISABLED")
   endif()
+  message(STATUS "ZSTD: ${Zstd_INCLUDE_DIR}")
   message(STATUS "ZLIB: ${ZLIB_INCLUDE_DIR}")
   message(STATUS "BZ2: ${BZIP2_INCLUDE_DIR}")
 
@@ -430,6 +483,7 @@ else()
     PRIVATE
     ${ZLIB_INCLUDE_DIR}
     ${BZIP2_INCLUDE_DIR}
+    ${Zstd_INCLUDE_DIR}
     ${CMAKE_BINARY_DIR}/external/libzip
     )
 
@@ -506,13 +560,15 @@ else()
 
     set(ZLIB_PATH "${CMAKE_BINARY_DIR}/libz-fat.a")
     set(BZ2_PATH "${CMAKE_BINARY_DIR}/libbz2-fat.a")
+    set(ZSTD_PATH "${CMAKE_BINARY_DIR}/libzstd-fat.a")
 
     if(ENABLE_XZ)
       set(LZMA_PATH "${CMAKE_BINARY_DIR}/liblzma-fat.a")
     endif()
 
     make_fat_archive("${ARTIFACTS_ROOT_DIR}/lib/libz.a" "${ARTIFACTS_OTHER_ROOT_DIR}/lib/libz.a" "${ZLIB_PATH}")
     make_fat_archive("${ARTIFACTS_ROOT_DIR}/lib/libbz2.a" "${ARTIFACTS_OTHER_ROOT_DIR}/lib/libbz2.a" "${BZ2_PATH}")
+    make_fat_archive("${ARTIFACTS_ROOT_DIR}/lib/libzstd.a" "${ARTIFACTS_OTHER_ROOT_DIR}/lib/libzstd.a" "${ZSTD_PATH}")
 
     if(ENABLE_XZ)
       make_fat_archive("${ARTIFACTS_ROOT_DIR}/lib/liblzma.a" "${ARTIFACTS_OTHER_ROOT_DIR}/lib/liblzma.a" "${LZMA_PATH}")
@@ -521,6 +577,7 @@ else()
     set(LIBS
       ${ZLIB_PATH}
       ${BZ2_PATH}
+      ${ZSTD_PATH}
       )
 
     if(ENABLE_XZ)
@@ -537,6 +594,7 @@ else()
       set(LIBS
         ${ARTIFACTS_ROOT_DIR}/lib/${ZLIB_NAME}
         ${ARTIFACTS_ROOT_DIR}/lib/bz2.lib
+        ${ARTIFACTS_ROOT_DIR}/lib/zstd.lib
         )
 
       if(ENABLE_XZ)
@@ -546,6 +604,7 @@ else()
       set(LIBS
         ${ARTIFACTS_ROOT_DIR}/lib/libz.a
         ${ARTIFACTS_ROOT_DIR}/lib/libbz2.a
+        ${ARTIFACTS_ROOT_DIR}/lib/libzstd.a
         )
 
       if(ENABLE_XZ)

LibZipSharp.UnitTest/ZipTests.cs

@@ -216,6 +216,7 @@ public void SmallTextFile ()
 
 		[TestCase (CompressionMethod.Deflate)]
 		[TestCase (CompressionMethod.Bzip2)]
+		[TestCase (CompressionMethod.ZSTD)]
 #if HAVE_XZ
 		[TestCase (CompressionMethod.XZ)]
 #endif

LibZipSharp.props

@@ -1,6 +1,6 @@
 <Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
-    <_LibZipSharpNugetVersion>2.0.0-alpha6</_LibZipSharpNugetVersion>
+    <_LibZipSharpNugetVersion>2.0.0-alpha7</_LibZipSharpNugetVersion>
     <_NativeBuildDir>$(MSBuildThisFileDirectory)lzsbuild</_NativeBuildDir>
     <_ExternalDir>$(MSBuildThisFileDirectory)external</_ExternalDir>
     <_MonoPosixNugetVersion>7.0.0-alpha8.21302.6</_MonoPosixNugetVersion>

LibZipSharp/Xamarin.Tools.Zip/Native.cs

@@ -40,6 +40,7 @@ public struct LZSVersions
 			public string libzip;
 			public string zlib;
 			public string zlibng;
+			public string zstd;
 			public string lzma;
 			public string libzipsharp;
 		};
@@ -109,6 +110,7 @@ public static Versions get_versions ()
 				LibZip = ret.libzip ?? String.Empty,
 				Zlib = ret.zlib ?? String.Empty,
 				ZlibNG = ret.zlibng ?? String.Empty,
+				ZStd = ret.zstd ?? String.Empty,
 				LZMA = ret.lzma ?? String.Empty,
 				LibZipSharp = ret.libzipsharp ?? String.Empty
 			};

LibZipSharp/Xamarin.Tools.Zip/Versions.cs

@@ -14,6 +14,10 @@ public string Zlib {
 			get; internal set;
 		}
 
+		public string ZStd {
+			get; internal set;
+		}
+
 		public string ZlibNG {
 			get; internal set;
 		}

azure-pipelines.yml

@@ -23,10 +23,6 @@ stages:
             echo "Hello"
             git submodule update --init --recursive
           displayName: "Update Submodules"
-        - script: |
-            cd external/libzip
-            git apply -v ../../libzip-changes.patch
-          displayName: "Apply Patches"
         - script: |
             build_windows.bat
           displayName: "Build"
@@ -59,9 +55,6 @@ stages:
             sudo apt -f -u install ninja-build -y
             git submodule update --init --recursive
           displayName: 'Install Tools'
-        - bash: |
-            patch --verbose -d external/libzip -p1 -l < libzip-changes.patch
-          displayName: 'Apply Patch'
         - bash: |
             ./build.sh -v
           displayName: 'Build Linux x64'
@@ -88,9 +81,6 @@ stages:
             brew install ninja
             git submodule update --init --recursive
           displayName: 'Install toolchain'
-        - bash: |
-            patch --verbose -d external/libzip -p1 -l < libzip-changes.patch
-          displayName: 'Apply Patch'
         - bash: |
             ./build.sh -v
             mkdir -p lzsbuild/lib/Linux