From a7c6df234664a5962df11706d00c6658ca82411a Mon Sep 17 00:00:00 2001 From: clux Date: Tue, 20 May 2025 21:52:37 +0100 Subject: [PATCH 1/4] remove openssl and libpq and an initial set of tests and dependencies Signed-off-by: clux --- Dockerfile.arm64 | 35 ++------------------- Dockerfile.x86_64 | 35 ++------------------- README.md | 21 +++---------- justfile | 6 ++-- test/dieselpgcrate/Cargo.toml | 8 ----- test/dieselpgcrate/src/main.rs | 52 ------------------------------- test/dieselsqlitecrate/Cargo.toml | 2 +- test/hypercrate/Cargo.toml | 12 ++++--- test/hypercrate/src/main.rs | 39 ++++++++++++----------- test/pqcrate/Cargo.toml | 8 ----- test/pqcrate/src/main.rs | 6 ---- test/sslcrate/Cargo.toml | 7 ----- test/sslcrate/src/main.rs | 12 ------- update_libs.py | 12 ------- 14 files changed, 40 insertions(+), 215 deletions(-) delete mode 100644 test/dieselpgcrate/Cargo.toml delete mode 100644 test/dieselpgcrate/src/main.rs delete mode 100644 test/pqcrate/Cargo.toml delete mode 100644 test/pqcrate/src/main.rs delete mode 100644 test/sslcrate/Cargo.toml delete mode 100644 test/sslcrate/src/main.rs diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index 81ba1b2..2784cca 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -6,8 +6,6 @@ LABEL maintainer="Eirik Albrigtsen " # - musl-dev, musl-tools - the musl toolchain # - curl, g++, make, pkgconf, cmake - for fetching and building third party libs # - ca-certificates - openssl + curl + peer verification of downloads -# - xutils-dev - for openssl makedepend -# - libssl-dev and libpq-dev - for dynamic linking during diesel_codegen build process # - git - cargo builds in user projects # - linux-headers-amd64 - needed for building openssl 1.1 (stretch only) # - file - needed by rustup.sh install @@ -25,9 +23,6 @@ RUN apt-get update && apt-get install -y \ curl \ pkgconf \ ca-certificates \ - xutils-dev \ - libssl-dev \ - libpq-dev \ automake \ autoconf \ libtool \ @@ -53,9 +48,7 @@ RUN chmod a+X /root # Convenience list of versions and variables for compilation later on # This helps continuing manually if anything breaks. -ENV SSL_VER="1.1.1w" \ - ZLIB_VER="1.3.1" \ - PQ_VER="11.12" \ +ENV ZLIB_VER="1.3.1" \ SQLITE_VER="3490100" \ PROTOBUF_VER="29.2" \ SCCACHE_VER="0.9.1" \ @@ -87,34 +80,13 @@ RUN mkdir $PREFIX && \ ln -s /usr/include/asm-generic /usr/include/aarch64-linux-musl/asm-generic && \ ln -s /usr/include/linux /usr/include/aarch64-linux-musl/linux -# Build zlib (used in openssl and pq) +# Build zlib (used in pq) RUN curl -sSL https://zlib.net/zlib-$ZLIB_VER.tar.gz | tar xz && \ cd zlib-$ZLIB_VER && \ CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure --static --prefix=$PREFIX && \ make -j$(nproc) && make install && \ cd .. && rm -rf zlib-$ZLIB_VER -# Build openssl (used in pq) -# Would like to use zlib here, but can't seem to get it to work properly -# TODO: fix so that it works -RUN curl -sSL https://www.openssl.org/source/openssl-$SSL_VER.tar.gz | tar xz && \ - cd openssl-$SSL_VER && \ - CFLAGS="-mno-outline-atomics" ./Configure no-zlib no-shared -fPIC --prefix=$PREFIX --openssldir=$PREFIX/ssl linux-aarch64 && \ - env C_INCLUDE_PATH=$PREFIX/include make depend 2> /dev/null && \ - make -j$(nproc) && make all install_sw && \ - cd .. && rm -rf openssl-$SSL_VER - -# Build libpq -RUN curl -sSL https://ftp.postgresql.org/pub/source/v$PQ_VER/postgresql-$PQ_VER.tar.gz | tar xz && \ - cd postgresql-$PQ_VER && \ - CC="musl-gcc -fPIE -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \ - --without-readline \ - --with-openssl \ - --prefix=$PREFIX --host=x86_64-unknown-linux-musl && \ - cd src/interfaces/libpq make -s -j$(nproc) all-static-lib && make -s install install-lib-static && \ - cd ../../bin/pg_config && make -j $(nproc) && make install && \ - cd .. && rm -rf postgresql-$PQ_VER - # Build libsqlite3 using same configuration as the alpine linux main/sqlite package RUN curl -sSL https://www.sqlite.org/2025/sqlite-autoconf-$SQLITE_VER.tar.gz | tar xz && \ cd sqlite-autoconf-$SQLITE_VER && \ @@ -138,11 +110,8 @@ ENV PATH=/root/.cargo/bin:$PREFIX/bin:$PATH \ CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUSTFLAGS="-Clink-self-contained=yes -Clinker=rust-lld -Ctarget-feature=+crt-static" \ PKG_CONFIG_ALLOW_CROSS=true \ PKG_CONFIG_ALL_STATIC=true \ - PQ_LIB_STATIC_AARCH64_UNKNOWN_LINUX_MUSL=true \ PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig \ PG_CONFIG_AARCH64_UNKNOWN_LINUX_GNU=/usr/bin/pg_config \ - OPENSSL_STATIC=true \ - OPENSSL_DIR=$PREFIX \ SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \ SSL_CERT_DIR=/etc/ssl/certs \ LIBZ_SYS_STATIC=1 \ diff --git a/Dockerfile.x86_64 b/Dockerfile.x86_64 index bca1881..38b8232 100644 --- a/Dockerfile.x86_64 +++ b/Dockerfile.x86_64 @@ -5,8 +5,6 @@ LABEL maintainer="Eirik Albrigtsen " # - musl-dev, musl-tools - the musl toolchain # - curl, g++, make, pkgconf, cmake - for fetching and building third party libs # - ca-certificates - openssl + curl + peer verification of downloads -# - xutils-dev - for openssl makedepend -# - libssl-dev and libpq-dev - for dynamic linking during diesel_codegen build process # - git - cargo builds in user projects # - linux-headers-amd64 - needed for building openssl 1.1 (stretch only) # - file - needed by rustup.sh install @@ -24,9 +22,6 @@ RUN apt-get update && apt-get install -y \ curl \ pkgconf \ ca-certificates \ - xutils-dev \ - libssl-dev \ - libpq-dev \ automake \ autoconf \ libtool \ @@ -52,9 +47,7 @@ RUN chmod a+X /root # Convenience list of versions and variables for compilation later on # This helps continuing manually if anything breaks. -ENV SSL_VER="1.1.1w" \ - ZLIB_VER="1.3.1" \ - PQ_VER="11.12" \ +ENV ZLIB_VER="1.3.1" \ SQLITE_VER="3490100" \ PROTOBUF_VER="29.2" \ SCCACHE_VER="0.9.1" \ @@ -86,34 +79,13 @@ RUN mkdir $PREFIX && \ ln -s /usr/include/asm-generic /usr/include/x86_64-linux-musl/asm-generic && \ ln -s /usr/include/linux /usr/include/x86_64-linux-musl/linux -# Build zlib (used in openssl and pq) +# Build zlib (used in pq) RUN curl -sSL https://zlib.net/zlib-$ZLIB_VER.tar.gz | tar xz && \ cd zlib-$ZLIB_VER && \ CC="musl-gcc -fPIC -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure --static --prefix=$PREFIX && \ make -j$(nproc) && make install && \ cd .. && rm -rf zlib-$ZLIB_VER -# Build openssl (used in pq) -# Would like to use zlib here, but can't seem to get it to work properly -# TODO: fix so that it works -RUN curl -sSL https://www.openssl.org/source/openssl-$SSL_VER.tar.gz | tar xz && \ - cd openssl-$SSL_VER && \ - ./Configure no-zlib no-shared -fPIC --prefix=$PREFIX --openssldir=$PREFIX/ssl linux-x86_64 && \ - env C_INCLUDE_PATH=$PREFIX/include make depend 2> /dev/null && \ - make -j$(nproc) && make all install_sw && \ - cd .. && rm -rf openssl-$SSL_VER - -# Build libpq -RUN curl -sSL https://ftp.postgresql.org/pub/source/v$PQ_VER/postgresql-$PQ_VER.tar.gz | tar xz && \ - cd postgresql-$PQ_VER && \ - CC="musl-gcc -fPIE -pie" LDFLAGS="-L$PREFIX/lib" CFLAGS="-I$PREFIX/include" ./configure \ - --without-readline \ - --with-openssl \ - --prefix=$PREFIX --host=x86_64-unknown-linux-musl && \ - cd src/interfaces/libpq make -s -j$(nproc) all-static-lib && make -s install install-lib-static && \ - cd ../../bin/pg_config && make -j $(nproc) && make install && \ - cd .. && rm -rf postgresql-$PQ_VER - # Build libsqlite3 using same configuration as the alpine linux main/sqlite package RUN curl -sSL https://www.sqlite.org/2025/sqlite-autoconf-$SQLITE_VER.tar.gz | tar xz && \ cd sqlite-autoconf-$SQLITE_VER && \ @@ -136,11 +108,8 @@ ENV PATH=/root/.cargo/bin:$PREFIX/bin:$PATH \ CARGO_BUILD_TARGET=x86_64-unknown-linux-musl \ PKG_CONFIG_ALLOW_CROSS=true \ PKG_CONFIG_ALL_STATIC=true \ - PQ_LIB_STATIC_X86_64_UNKNOWN_LINUX_MUSL=true \ PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig \ PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU=/usr/bin/pg_config \ - OPENSSL_STATIC=true \ - OPENSSL_DIR=$PREFIX \ SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \ SSL_CERT_DIR=/etc/ssl/certs \ LIBZ_SYS_STATIC=1 \ diff --git a/README.md b/README.md index d3a14d8..1a5194e 100644 --- a/README.md +++ b/README.md @@ -50,14 +50,15 @@ For pinned, or historical builds, see the [available tags on dockerhub](https:// The following system libraries are compiled against `musl-gcc`: -- [x] openssl ([openssl crate](https://github.com/sfackler/rust-openssl)) -- [x] pq ([pq-sys crate](https://github.com/sgrif/pq-sys) used by [diesel](https://github.com/diesel-rs/diesel)) - [x] sqlite3 ([libsqlite3-sys crate](https://github.com/jgallagher/rusqlite/tree/master/libsqlite3-sys) used by [diesel](https://github.com/diesel-rs/diesel)) -- [x] zlib (used by pq and openssl) +- [x] zlib We **[try](https://github.com/clux/muslrust/blob/main/update_libs.py)** to keep these up to date. -NB: C library for `curl` has been removed in newer tags from 2025. See [#96](https://github.com/clux/muslrust/issues/96). +Removed Libraries; +- `openssl` has been removed in 2025. See [#153](https://github.com/clux/muslrust/issues/153). +- `curl` has been removed in 2025. See [#96](https://github.com/clux/muslrust/issues/96). +- `pq` has been removed in 2025. See [#81](https://github.com/clux/muslrust/issues/81) ## Developing @@ -77,7 +78,6 @@ Before we push a new version of muslrust we [test](https://github.com/clux/muslr - [x] [diesel](https://crates.io/crates/diesel) - [x] [hyper](https://crates.io/crates/hyper) - [x] [rustls](https://crates.io/crates/rustls) -- [x] [openssl](https://crates.io/crates/openssl) - [x] [flate2](https://crates.io/crates/flate2) - [x] [rand](https://crates.io/crates/rand) @@ -139,17 +139,6 @@ static GLOBAL: Jemalloc = Jemalloc; ## Troubleshooting -### SSL Verification - -You might need to point `openssl` at the location of your certificates **explicitly** to avoid certificate errors on https requests. - -```sh -export SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt -export SSL_CERT_DIR=/etc/ssl/certs -``` - -These can be [hardcoded in your Dockerfile](https://docs.docker.com/engine/reference/builder/#env), or you can rely on the [openssl-probe crate](https://crates.io/crates/openssl-probe) to detect the cert location. You should not have to do this if you are using the static variants of `distroless` or `chainguard`. - ### Diesel and PQ builds Works with the older version of libpq we bundle (see [#81](https://github.com/clux/muslrust/issues/81)). See the [test/dieselpgcrate](./test/dieselpgcrate) for specifics. diff --git a/justfile b/justfile index c92807b..7c4ab0d 100644 --- a/justfile +++ b/justfile @@ -32,7 +32,7 @@ _ti crate: just _t_{{ os() }}_{{ arch() }} {{crate}} # when running locally we can use one of these instead of _t -_t_linux_amd64 crate: +_t_linux_x86_64 crate: #!/bin/bash export PLATFORM="linux/amd64" export TARGET_DIR="x86_64-unknown-linux-musl" @@ -46,9 +46,9 @@ _t_macos_aarch64 crate: ./test.sh {{crate}} # Test all crates against built container locally -test: (_ti "plain") (_ti "ssl") (_ti "rustls") (_ti "pq") (_ti "serde") (_ti "zlib") (_ti "hyper") (_ti "dieselpg") (_ti "dieselsqlite") +test: (_ti "plain") (_ti "rustls") (_ti "serde") (_ti "zlib") (_ti "hyper") (_ti "dieselsqlite") # Test all crates against built container in ci (inheriting set PLATFORM/TARGET_DIR/AR vars) -test-ci: (_t "plain") (_t "ssl") (_t "rustls") (_t "pq") (_t "serde") (_t "zlib") (_t "hyper") (_t "dieselpg") (_t "dieselsqlite") +test-ci: (_t "plain") (_t "rustls") (_t "serde") (_t "zlib") (_t "hyper") (_t "dieselsqlite") # Cleanup everything clean: clean-docker clean-tests diff --git a/test/dieselpgcrate/Cargo.toml b/test/dieselpgcrate/Cargo.toml deleted file mode 100644 index da90411..0000000 --- a/test/dieselpgcrate/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -authors = ["clux "] -name = "dieselpgcrate" -version = "0.1.0" - -[dependencies] -diesel = { version = "2.1.*", features = ["postgres"] } -openssl = "*" diff --git a/test/dieselpgcrate/src/main.rs b/test/dieselpgcrate/src/main.rs deleted file mode 100644 index ef54577..0000000 --- a/test/dieselpgcrate/src/main.rs +++ /dev/null @@ -1,52 +0,0 @@ -// The order of these extern crate lines matter for ssl! -extern crate openssl; -#[macro_use] -extern crate diesel; -// openssl must be included before diesel atm. - -mod schema { - table! { - posts (id) { - id -> Int4, - title -> Varchar, - body -> Text, - published -> Bool, - } - } -} - -mod models { - use schema::posts; - #[derive(Queryable)] - pub struct Post { - pub id: i32, - pub title: String, - pub body: String, - pub published: bool, - } - - // apparently this can be done without heap storage, but lifetimes spread far.. - #[derive(Insertable)] - #[diesel(table_name = posts)] - pub struct NewPost { - pub title: String, - pub body: String, - } -} - -use diesel::pg::PgConnection; -use diesel::prelude::*; - -fn main() { - let database_url = std::env::var("DATABASE_URL") - .unwrap_or("postgres://localhost?connect_timeout=1&sslmode=require".into()); - match PgConnection::establish(&database_url) { - Err(e) => { - println!("Should fail to connect here:"); - println!("{}", e); - } - Ok(_) => { - unreachable!(); - } - } -} diff --git a/test/dieselsqlitecrate/Cargo.toml b/test/dieselsqlitecrate/Cargo.toml index 266daf3..ab2d224 100644 --- a/test/dieselsqlitecrate/Cargo.toml +++ b/test/dieselsqlitecrate/Cargo.toml @@ -4,4 +4,4 @@ name = "dieselsqlitecrate" version = "0.1.0" [dependencies] -diesel = { version = "2.1.*", features = ["sqlite"] } +diesel = { version = "2.2.*", features = ["sqlite"] } diff --git a/test/hypercrate/Cargo.toml b/test/hypercrate/Cargo.toml index 341e62b..8f3f8a1 100644 --- a/test/hypercrate/Cargo.toml +++ b/test/hypercrate/Cargo.toml @@ -1,10 +1,12 @@ [package] authors = ["clux "] name = "hypercrate" -version = "0.1.0" -edition = "2018" +version = "0.2.0" +edition = "2024" [dependencies] -hyper = { version = "0.14", features=["client","http1"] } -tokio = { version = "1.12.0", features=["macros", "io-std", "io-util"] } -hyper-tls = "0.5.0" +hyper = { version = "1.6", features=["client", "http1"] } +tokio = { version = "1.45.0", features=["macros", "io-std", "io-util", "rt-multi-thread"] } +hyper-rustls = "0.27.5" +http-body-util = "0.1.3" +hyper-util = "0.1.12" diff --git a/test/hypercrate/src/main.rs b/test/hypercrate/src/main.rs index 14185bf..9ede8f3 100644 --- a/test/hypercrate/src/main.rs +++ b/test/hypercrate/src/main.rs @@ -1,26 +1,27 @@ -use hyper_tls::HttpsConnector; -use hyper::{body::HttpBody as _, Client}; -use tokio::io::{self, AsyncWriteExt as _}; +//- Example from https://docs.rs/hyper-rustls/latest/hyper_rustls/ +use http_body_util::Empty; +use hyper::body::Bytes; +use hyper::http::StatusCode; +use hyper_util::client::legacy::Client; +use hyper_util::rt::TokioExecutor; -#[tokio::main(flavor = "current_thread")] -async fn main() -> Result<(), Box>{ - // set SSL_CERT location - see issue #5 - // normally you'd want to set this in your container - // but for plain bin distribution and this test, we set it here - std::env::set_var("SSL_CERT_FILE", "/etc/ssl/certs/ca-certificates.crt"); +#[tokio::main] +async fn main() -> Result<(), Box> { + let url = ("https://raw.githubusercontent.com/clux/muslrust/master/README.md") + .parse() + .unwrap(); - let url = "https://raw.githubusercontent.com/clux/muslrust/master/README.md"; + let https = hyper_rustls::HttpsConnectorBuilder::new() + .with_native_roots() + .expect("no native root CA certificates found") + .https_only() + .enable_http1() + .build(); - let https = HttpsConnector::new(); - let client = Client::builder().build::<_, hyper::Body>(https); + let client: Client<_, Empty> = Client::builder(TokioExecutor::new()).build(https); - let mut res = client.get(url.parse()?).await?; - assert_eq!(res.status(), 200); - - while let Some(next) = res.data().await { - let chunk = next?; - io::stdout().write_all(&chunk).await?; - } + let res = client.get(url).await.unwrap(); + assert_eq!(res.status(), StatusCode::OK); Ok(()) } diff --git a/test/pqcrate/Cargo.toml b/test/pqcrate/Cargo.toml deleted file mode 100644 index 6241052..0000000 --- a/test/pqcrate/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -authors = ["clux "] -name = "pqcrate" -version = "0.1.0" - -[dependencies] -pq-sys = "0.5" -openssl = "*" diff --git a/test/pqcrate/src/main.rs b/test/pqcrate/src/main.rs deleted file mode 100644 index 9249259..0000000 --- a/test/pqcrate/src/main.rs +++ /dev/null @@ -1,6 +0,0 @@ -extern crate openssl; // needed to avoid link errors even if we don't use it directly -extern crate pq_sys; - -fn main() { - unsafe{ pq_sys::PQinitSSL(1); } -} diff --git a/test/sslcrate/Cargo.toml b/test/sslcrate/Cargo.toml deleted file mode 100644 index 755e4cf..0000000 --- a/test/sslcrate/Cargo.toml +++ /dev/null @@ -1,7 +0,0 @@ -[package] -authors = ["clux "] -name = "sslcrate" -version = "0.1.0" - -[dependencies] -openssl = "*" diff --git a/test/sslcrate/src/main.rs b/test/sslcrate/src/main.rs deleted file mode 100644 index f51b64b..0000000 --- a/test/sslcrate/src/main.rs +++ /dev/null @@ -1,12 +0,0 @@ -extern crate openssl; -use std::str; -use openssl::hash::{hash, MessageDigest}; - - -fn main() { - let data: &[u8] = b"Hello, world"; - let digest = hash(MessageDigest::sha256(), &data); - - println!("{}", str::from_utf8(data).ok().unwrap()); - println!("hash: {:?}", digest); -} diff --git a/update_libs.py b/update_libs.py index 5c4e640..895c334 100755 --- a/update_libs.py +++ b/update_libs.py @@ -22,16 +22,6 @@ import re -def convert_openssl_version(version): - """Convert OpenSSL package versions to match upstream's format - - >>> convert_openssl_version('1.0.2.o') - '1.0.2o' - """ - - return re.sub(r'(.+)\.([a-z])', r'\1\2', version) - - def convert_sqlite_version(version): """Convert SQLite package versions to match upstream's format @@ -82,9 +72,7 @@ def rustup_version(): if __name__ == '__main__': PACKAGES = { - #'PQ': pkgver('postgresql-old-upgrade'), # see https://github.com/clux/muslrust/issues/81 'SQLITE': convert_sqlite_version(pkgver('sqlite')), - 'SSL': convert_openssl_version(pkgver('openssl-1.1')), 'SCCACHE': pkgver('sccache'), 'PROTOBUF': pkgver('protobuf'), 'ZLIB': pkgver('zlib'), From dc9ea1cc640006b9dc7e7629ca76b0695f05e8e4 Mon Sep 17 00:00:00 2001 From: clux Date: Tue, 20 May 2025 21:54:45 +0100 Subject: [PATCH 2/4] minimize dockerfile diff Signed-off-by: clux --- Dockerfile.x86_64 | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.x86_64 b/Dockerfile.x86_64 index 38b8232..9b78cde 100644 --- a/Dockerfile.x86_64 +++ b/Dockerfile.x86_64 @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 FROM ubuntu:noble LABEL maintainer="Eirik Albrigtsen " From 3ffb724440405ec32738cc14f7fe8345bd3b17bc Mon Sep 17 00:00:00 2001 From: clux Date: Tue, 20 May 2025 22:01:14 +0100 Subject: [PATCH 3/4] remove old openssl and pq hacks Signed-off-by: clux --- Dockerfile.arm64 | 22 +--------------------- Dockerfile.x86_64 | 20 -------------------- 2 files changed, 1 insertion(+), 41 deletions(-) diff --git a/Dockerfile.arm64 b/Dockerfile.arm64 index 2784cca..af3c1eb 100644 --- a/Dockerfile.arm64 +++ b/Dockerfile.arm64 @@ -5,9 +5,8 @@ LABEL maintainer="Eirik Albrigtsen " # Required packages: # - musl-dev, musl-tools - the musl toolchain # - curl, g++, make, pkgconf, cmake - for fetching and building third party libs -# - ca-certificates - openssl + curl + peer verification of downloads +# - ca-certificates - peer verification of downloads # - git - cargo builds in user projects -# - linux-headers-amd64 - needed for building openssl 1.1 (stretch only) # - file - needed by rustup.sh install # - automake autoconf libtool - support crates building C deps as part cargo build # NB: does not include cmake atm @@ -71,15 +70,6 @@ RUN curl -sSL https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VE chmod +x /usr/local/bin/sccache && \ rm -rf sccache-v${SCCACHE_VER}-*-unknown-linux-musl -# Set up a prefix for musl build libraries, make the linker's job of finding them easier -# Primarily for the benefit of postgres. -# Lastly, link some linux-headers for openssl 1.1 (not used herein) -RUN mkdir $PREFIX && \ - echo "$PREFIX/lib" >> /etc/ld-musl-aarch64.path && \ - ln -s /usr/include/aarch64-linux-gnu/asm /usr/include/aarch64-linux-musl/asm && \ - ln -s /usr/include/asm-generic /usr/include/aarch64-linux-musl/asm-generic && \ - ln -s /usr/include/linux /usr/include/aarch64-linux-musl/linux - # Build zlib (used in pq) RUN curl -sSL https://zlib.net/zlib-$ZLIB_VER.tar.gz | tar xz && \ cd zlib-$ZLIB_VER && \ @@ -96,14 +86,6 @@ RUN curl -sSL https://www.sqlite.org/2025/sqlite-autoconf-$SQLITE_VER.tar.gz | t make && make install && \ cd .. && rm -rf sqlite-autoconf-$SQLITE_VER -# SSL cert directories get overridden by --prefix and --openssldir -# and they do not match the typical host configurations. -# The SSL_CERT_* vars fix this, but only when inside this container -# musl-compiled binary must point SSL at the correct certs (muslrust/issues/5) elsewhere -# Postgres bindings need vars so that diesel_codegen.so uses the GNU deps at build time -# but finally links with the static libpq.a at the end. -# It needs the non-musl pg_config to set this up with libpq-dev (depending on libssl-dev) -# See https://github.com/sgrif/pq-sys/pull/18 ENV PATH=/root/.cargo/bin:$PREFIX/bin:$PATH \ RUSTUP_HOME=/root/.rustup \ CARGO_BUILD_TARGET=aarch64-unknown-linux-musl \ @@ -112,8 +94,6 @@ ENV PATH=/root/.cargo/bin:$PREFIX/bin:$PATH \ PKG_CONFIG_ALL_STATIC=true \ PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig \ PG_CONFIG_AARCH64_UNKNOWN_LINUX_GNU=/usr/bin/pg_config \ - SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \ - SSL_CERT_DIR=/etc/ssl/certs \ LIBZ_SYS_STATIC=1 \ DEBIAN_FRONTEND=noninteractive \ TZ=Etc/UTC diff --git a/Dockerfile.x86_64 b/Dockerfile.x86_64 index 9b78cde..aae5da5 100644 --- a/Dockerfile.x86_64 +++ b/Dockerfile.x86_64 @@ -7,7 +7,6 @@ LABEL maintainer="Eirik Albrigtsen " # - curl, g++, make, pkgconf, cmake - for fetching and building third party libs # - ca-certificates - openssl + curl + peer verification of downloads # - git - cargo builds in user projects -# - linux-headers-amd64 - needed for building openssl 1.1 (stretch only) # - file - needed by rustup.sh install # - automake autoconf libtool - support crates building C deps as part cargo build # NB: does not include cmake atm @@ -71,15 +70,6 @@ RUN curl -sSL https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VE chmod +x /usr/local/bin/sccache && \ rm -rf sccache-v${SCCACHE_VER}-*-unknown-linux-musl -# Set up a prefix for musl build libraries, make the linker's job of finding them easier -# Primarily for the benefit of postgres. -# Lastly, link some linux-headers for openssl 1.1 (not used herein) -RUN mkdir $PREFIX && \ - echo "$PREFIX/lib" >> /etc/ld-musl-x86_64.path && \ - ln -s /usr/include/x86_64-linux-gnu/asm /usr/include/x86_64-linux-musl/asm && \ - ln -s /usr/include/asm-generic /usr/include/x86_64-linux-musl/asm-generic && \ - ln -s /usr/include/linux /usr/include/x86_64-linux-musl/linux - # Build zlib (used in pq) RUN curl -sSL https://zlib.net/zlib-$ZLIB_VER.tar.gz | tar xz && \ cd zlib-$ZLIB_VER && \ @@ -96,14 +86,6 @@ RUN curl -sSL https://www.sqlite.org/2025/sqlite-autoconf-$SQLITE_VER.tar.gz | t make && make install && \ cd .. && rm -rf sqlite-autoconf-$SQLITE_VER -# SSL cert directories get overridden by --prefix and --openssldir -# and they do not match the typical host configurations. -# The SSL_CERT_* vars fix this, but only when inside this container -# musl-compiled binary must point SSL at the correct certs (muslrust/issues/5) elsewhere -# Postgres bindings need vars so that diesel_codegen.so uses the GNU deps at build time -# but finally links with the static libpq.a at the end. -# It needs the non-musl pg_config to set this up with libpq-dev (depending on libssl-dev) -# See https://github.com/sgrif/pq-sys/pull/18 ENV PATH=/root/.cargo/bin:$PREFIX/bin:$PATH \ RUSTUP_HOME=/root/.rustup \ CARGO_BUILD_TARGET=x86_64-unknown-linux-musl \ @@ -111,8 +93,6 @@ ENV PATH=/root/.cargo/bin:$PREFIX/bin:$PATH \ PKG_CONFIG_ALL_STATIC=true \ PKG_CONFIG_PATH=$PREFIX/lib/pkgconfig \ PG_CONFIG_X86_64_UNKNOWN_LINUX_GNU=/usr/bin/pg_config \ - SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \ - SSL_CERT_DIR=/etc/ssl/certs \ LIBZ_SYS_STATIC=1 \ DEBIAN_FRONTEND=noninteractive \ TZ=Etc/UTC From c844e0fb783443dfadd035ab88f5d5e0254e0ec4 Mon Sep 17 00:00:00 2001 From: clux Date: Tue, 20 May 2025 22:55:12 +0100 Subject: [PATCH 4/4] remove a duplicate test and cross-document to blackdex Signed-off-by: clux --- README.md | 36 +++++++------------ justfile | 4 +-- test/{hypercrate => hypertlscrate}/Cargo.toml | 2 +- .../{hypercrate => hypertlscrate}/src/main.rs | 0 test/plaincrate/Cargo.toml | 1 + test/rustlscrate/Cargo.toml | 13 ------- test/rustlscrate/src/main.rs | 21 ----------- test/serdecrate/Cargo.toml | 2 +- test/zlibcrate/Cargo.toml | 1 + 9 files changed, 19 insertions(+), 61 deletions(-) rename test/{hypercrate => hypertlscrate}/Cargo.toml (93%) rename test/{hypercrate => hypertlscrate}/src/main.rs (100%) delete mode 100644 test/rustlscrate/Cargo.toml delete mode 100644 test/rustlscrate/src/main.rs diff --git a/README.md b/README.md index 1a5194e..38ae795 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,13 @@ [![stable](https://github.com/clux/muslrust/actions/workflows/stable.yml/badge.svg)](https://github.com/clux/muslrust/actions/workflows/stable.yml) [![docker pulls](https://img.shields.io/docker/pulls/clux/muslrust.svg)](https://hub.docker.com/r/clux/muslrust/tags) -A docker environment for building **static** rust binaries for `x86_64` and `arm64` **linux** environments using **[musl](https://musl.libc.org/)**. Built daily via [github actions](https://github.com/clux/muslrust/actions). +A docker environment for building **static** rust binaries for `x86_64` and `arm64` environments using **[musl](https://musl.libc.org/)**. Built daily via [github actions](https://github.com/clux/muslrust/actions). -Binaries compiled with `muslrust` are **light-weight**, call straight into the kernel without other dynamic system library dependencies, can be shipped to most linux distributions without compatibility issues, and can be inserted as-is into lightweight docker images such as [static distroless](https://github.com/GoogleContainerTools/distroless/blob/main/base/README.md), [scratch](https://hub.docker.com/_/scratch), or [alpine](https://hub.docker.com/_/alpine). +Binaries compiled with `muslrust` are **light-weight**, call straight into the kernel without other dynamic system library dependencies, can be shipped to most distributions without compatibility issues, and can be inserted as-is into lightweight docker images such as [static distroless](https://github.com/GoogleContainerTools/distroless/blob/main/base/README.md), [scratch](https://hub.docker.com/_/scratch), or [alpine](https://hub.docker.com/_/alpine). The goal is to **simplify** the creation of small and **efficient cloud containers**, or **stand-alone linux binary releases**. -This image includes popular [C libraries](#c-libraries) compiled with `musl-gcc`, enabling static builds even when these libraries are used. +This image includes some hard-to-avoid [C libraries](#c-libraries) compiled with `musl-gcc`, enabling static builds even when these libraries are used. ## Usage @@ -50,16 +50,21 @@ For pinned, or historical builds, see the [available tags on dockerhub](https:// The following system libraries are compiled against `musl-gcc`: -- [x] sqlite3 ([libsqlite3-sys crate](https://github.com/jgallagher/rusqlite/tree/master/libsqlite3-sys) used by [diesel](https://github.com/diesel-rs/diesel)) -- [x] zlib +- sqlite3 ([libsqlite3-sys crate](https://github.com/jgallagher/rusqlite/tree/master/libsqlite3-sys) used by [diesel](https://github.com/diesel-rs/diesel)) +- zlib -We **[try](https://github.com/clux/muslrust/blob/main/update_libs.py)** to keep these up to date. +Note that these libraries **may be removed** if sensible and popular Rust crates can replace them in the future. + +In the mean time, we **[try](https://github.com/clux/muslrust/blob/main/update_libs.py)** to keep these up to date. Removed Libraries; + - `openssl` has been removed in 2025. See [#153](https://github.com/clux/muslrust/issues/153). - `curl` has been removed in 2025. See [#96](https://github.com/clux/muslrust/issues/96). - `pq` has been removed in 2025. See [#81](https://github.com/clux/muslrust/issues/81) +Consider [blackdex/rust-musl](https://github.com/BlackDex/rust-musl) for `openssl`, `curl` and `pq`. + ## Developing Clone, tweak, build, and run tests: @@ -75,9 +80,9 @@ just test Before we push a new version of muslrust we [test](https://github.com/clux/muslrust/blob/main/test.sh#L4-L17) to ensure that we can use and statically link: - [x] [serde](https://crates.io/crates/serde) -- [x] [diesel](https://crates.io/crates/diesel) -- [x] [hyper](https://crates.io/crates/hyper) +- [x] [diesel](https://crates.io/crates/diesel) (using sqlite) - [x] [rustls](https://crates.io/crates/rustls) +- [x] [hyper](https://crates.io/crates/hyper) (using hyper-rustls and rustls's default crypto backend) - [x] [flate2](https://crates.io/crates/flate2) - [x] [rand](https://crates.io/crates/rand) @@ -139,21 +144,6 @@ static GLOBAL: Jemalloc = Jemalloc; ## Troubleshooting -### Diesel and PQ builds - -Works with the older version of libpq we bundle (see [#81](https://github.com/clux/muslrust/issues/81)). See the [test/dieselpgcrate](./test/dieselpgcrate) for specifics. - -For stuff like `infer_schema!` to work you need to explicitly pass on `-e DATABASE_URL=$DATABASE_URL` to the `docker run`. It's probably easier to just make `diesel print-schema > src/schema.rs` part of your migration setup though. - -Note that diesel compiles with `openssl` statically since `1.34.0`, so you need to include the `openssl` crate **before** `diesel` due to [pq-sys#25](https://github.com/sgrif/pq-sys/issues/25): - -```rs -extern crate openssl; -#[macro_use] extern crate diesel; -``` - -This is true even if you connect without `sslmode=require`. - ### Filesystem permissions on local builds When building locally, the permissions of the musl parts of the `./target` artifacts dir will be owned by `root` and requires `sudo rm -rf target/` to clear. This is an [intended](https://github.com/clux/muslrust/issues/65) complexity tradeoff with user builds. diff --git a/justfile b/justfile index 7c4ab0d..39e44db 100644 --- a/justfile +++ b/justfile @@ -46,9 +46,9 @@ _t_macos_aarch64 crate: ./test.sh {{crate}} # Test all crates against built container locally -test: (_ti "plain") (_ti "rustls") (_ti "serde") (_ti "zlib") (_ti "hyper") (_ti "dieselsqlite") +test: (_ti "plain") (_ti "serde") (_ti "zlib") (_ti "hypertls") (_ti "dieselsqlite") # Test all crates against built container in ci (inheriting set PLATFORM/TARGET_DIR/AR vars) -test-ci: (_t "plain") (_t "rustls") (_t "serde") (_t "zlib") (_t "hyper") (_t "dieselsqlite") +test-ci: (_t "plain") (_t "serde") (_t "zlib") (_t "hypertls") (_t "dieselsqlite") # Cleanup everything clean: clean-docker clean-tests diff --git a/test/hypercrate/Cargo.toml b/test/hypertlscrate/Cargo.toml similarity index 93% rename from test/hypercrate/Cargo.toml rename to test/hypertlscrate/Cargo.toml index 8f3f8a1..e8170f1 100644 --- a/test/hypercrate/Cargo.toml +++ b/test/hypertlscrate/Cargo.toml @@ -1,6 +1,6 @@ [package] authors = ["clux "] -name = "hypercrate" +name = "hypertlscrate" version = "0.2.0" edition = "2024" diff --git a/test/hypercrate/src/main.rs b/test/hypertlscrate/src/main.rs similarity index 100% rename from test/hypercrate/src/main.rs rename to test/hypertlscrate/src/main.rs diff --git a/test/plaincrate/Cargo.toml b/test/plaincrate/Cargo.toml index 3fa1102..2f2b5a2 100644 --- a/test/plaincrate/Cargo.toml +++ b/test/plaincrate/Cargo.toml @@ -2,6 +2,7 @@ authors = ["clux "] name = "plaincrate" version = "0.1.0" +edition = "2024" [dependencies] rand = "*" diff --git a/test/rustlscrate/Cargo.toml b/test/rustlscrate/Cargo.toml deleted file mode 100644 index 50714dc..0000000 --- a/test/rustlscrate/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -authors = ["clux "] -name = "rustlscrate" -version = "0.1.0" -edition = "2021" - -[dependencies] -http-body-util = "0.1.2" -hyper = { version = "1.6.0", features = ["client"] } -hyper-rustls = "0.27.5" -hyper-util = "0.1.10" -rustls = "0.23" -tokio = { version = "1.43.0", features = ["rt-multi-thread", "macros"] } diff --git a/test/rustlscrate/src/main.rs b/test/rustlscrate/src/main.rs deleted file mode 100644 index f557a70..0000000 --- a/test/rustlscrate/src/main.rs +++ /dev/null @@ -1,21 +0,0 @@ -use http_body_util::Empty; -use hyper::body::Bytes; -use hyper::http::StatusCode; -use hyper_util::client::legacy::Client; -use hyper_util::rt::TokioExecutor; - -#[tokio::main] -async fn main() { - let url = ("https://hyper.rs").parse().unwrap(); - let https = hyper_rustls::HttpsConnectorBuilder::new() - .with_native_roots() - .expect("no native root CA certificates found") - .https_only() - .enable_http1() - .build(); - - let client: Client<_, Empty> = Client::builder(TokioExecutor::new()).build(https); - - let res = client.get(url).await.unwrap(); - assert_eq!(res.status(), StatusCode::OK); -} diff --git a/test/serdecrate/Cargo.toml b/test/serdecrate/Cargo.toml index e98b8e9..1efe636 100644 --- a/test/serdecrate/Cargo.toml +++ b/test/serdecrate/Cargo.toml @@ -2,7 +2,7 @@ authors = ["clux "] name = "serdecrate" version = "0.1.0" -edition = "2021" +edition = "2024" [dependencies] serde = "1.0.15" diff --git a/test/zlibcrate/Cargo.toml b/test/zlibcrate/Cargo.toml index 4c1e380..f8623a7 100644 --- a/test/zlibcrate/Cargo.toml +++ b/test/zlibcrate/Cargo.toml @@ -2,6 +2,7 @@ authors = ["clux "] name = "zlibcrate" version = "0.1.0" +edition = "2024" [dependencies] flate2 = "*"