From c7a6d563628b75dbee17f351f7c185365be9f4ce Mon Sep 17 00:00:00 2001 From: Giedrius Date: Thu, 6 Oct 2022 09:58:57 +0300 Subject: [PATCH 1/2] Added PARALLEL_JOBS env var to speed up building, default value is 4 --- .common | 3 ++- bootstrap.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.common b/.common index 2a747523..5f8b4967 100644 --- a/.common +++ b/.common @@ -9,6 +9,7 @@ set -e # environment variables WORKDIR=${WORKDIR:=~/mod-workdir} +PARALLEL_JOBS=${PARALLEL_JOBS:-4} ####################################################################################################################### # Colored print functions @@ -104,7 +105,7 @@ BUILDROOT_VERSION=buildroot-2016.02 BUILDROOT_FILE=${BUILDROOT_VERSION}.tar.bz2 SOURCE_DIR=$("${readlink}" -f $(dirname $0)) -BR2_MAKE="make O=${WORKDIR}/${PLATFORM} BR2_EXTERNAL=${SOURCE_DIR}/plugins-dep" +BR2_MAKE="make O=${WORKDIR}/${PLATFORM} BR2_EXTERNAL=${SOURCE_DIR}/plugins-dep -j${PARALLEL_JOBS}" BR2_TARGET=${WORKDIR}/${PLATFORM}/target ####################################################################################################################### diff --git a/bootstrap.sh b/bootstrap.sh index 959ae57b..38d7c0e2 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -108,7 +108,7 @@ fi if [ ! -f .stamp_built2 ]; then sed -i -e 's/.PHONY: $(PHONY)/.PHONY: build $(PHONY)/' ct-ng - ./ct-ng build + CT_JOBS=$PARALLEL_JOBS ./ct-ng build touch .stamp_built2 fi From d8e00327ceb49b60d501360a98b9016a87b56185 Mon Sep 17 00:00:00 2001 From: Giedrius Date: Wed, 14 Dec 2022 14:46:22 +0200 Subject: [PATCH 2/2] Docker reworked to use docker compose --- .docker | 3 ++ .dockerignore | 6 ++++ .env | 2 ++ .gitignore | 1 + Dockerfile | 31 ++++++++++++++++++++ docker-arm64/Dockerfile | 56 ------------------------------------ docker-build | 28 ++++++++++++++++++ docker-build-all | 20 +++++++++++++ docker-compose.yml | 61 +++++++++++++++++++++++++++++++++++++++ docker-init | 13 +++++++++ docker-mount.sh | 31 -------------------- docker-mpb-push-all | 16 +++++++++++ docker/Dockerfile | 64 ----------------------------------------- 13 files changed, 181 insertions(+), 151 deletions(-) create mode 100644 .docker create mode 100644 .dockerignore create mode 100644 .env create mode 100644 Dockerfile delete mode 100644 docker-arm64/Dockerfile create mode 100755 docker-build create mode 100755 docker-build-all create mode 100644 docker-compose.yml create mode 100755 docker-init delete mode 100755 docker-mount.sh create mode 100755 docker-mpb-push-all delete mode 100644 docker/Dockerfile diff --git a/.docker b/.docker new file mode 100644 index 00000000..073c3dda --- /dev/null +++ b/.docker @@ -0,0 +1,3 @@ +: "${DOCKER_PLATFORMS:=x86_64 modduo-new}" + +. ./.env diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..d8e79b04 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,6 @@ +.git/* +docker-compose.yml +docker-workdir/* +lv2-data/* +lv2-data-creative-commons/* +plugins/* diff --git a/.env b/.env new file mode 100644 index 00000000..c657fe7b --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +DOCKER_ORGANIZATION="blokas" +DOCKER_IMG_PREFIX="mod_plugin_builder-" diff --git a/.gitignore b/.gitignore index f2942d9f..072d1a0c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ plugins/package/eg-amp-lv2/source/eg-amp.lv2/amp.so plugins/package/eg-amp-lv2/source/eg-amp.lv2/manifest.ttl plugins/package/eg-amp-lv2/eg-amp.lv2/eg-amp.lv2/amp.so plugins/package/eg-amp-lv2/eg-amp.lv2/eg-amp.lv2/manifest.ttl +docker-workdir/* diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..165e5bc0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:22.04 +LABEL maintainer="The Maintainer " + +ENV LANG C.UTF-8 +ENV DEBIAN_FRONTEND noninteractive +ENV USER builder +ENV HOME /home/$USER + +ARG REQUIRED_PKGS=" \ + bash bash-completion bc binutils bison build-essential bzip2 cpio cvs file flex gawk git help2man libtool-bin mercurial \ + ncurses-dev nano subversion pkg-config python3-dev python-is-python3 python3-setuptools rsync sudo tar texinfo unzip wget \ + " + +RUN apt-get update && apt-get upgrade -qy && apt-get install -qy $REQUIRED_PKGS && apt-get clean && \ + useradd -d $HOME -m -G sudo $USER && echo "$USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/99-$USER && chmod 0440 /etc/sudoers.d/99-$USER && \ + echo "source /etc/bash_completion" >> $HOME/.bashrc + +USER $USER +COPY . $HOME/mod-plugin-builder +WORKDIR $HOME/mod-plugin-builder + +# Must not be named "PLATFORM" in order not to clash with buildroot. +ARG MPB_PLATFORM + +# Below step is useful for debugging buildroot build failures, +# so it can be debugged without rebuilding the toolchain every time. +#RUN ./bootstrap.sh $MPB_PLATFORM toolchain + +RUN ./bootstrap.sh $MPB_PLATFORM && ./.clean-install.sh $MPB_PLATFORM + +CMD ["bash"] diff --git a/docker-arm64/Dockerfile b/docker-arm64/Dockerfile deleted file mode 100644 index ccfe39ab..00000000 --- a/docker-arm64/Dockerfile +++ /dev/null @@ -1,56 +0,0 @@ -FROM ubuntu:20.04 -LABEL maintainer="Filipe Coelho " -ENV DEBIAN_FRONTEND noninteractive - -# platform argument must be provided, e.g. --build-arg platform=modduo -ARG platform=moddwarf - -# enable armhf -RUN dpkg --add-architecture armhf - -RUN echo "deb [arch=armhf,arm64] http://ports.ubuntu.com/ubuntu-ports focal main restricted universe multiverse" > /etc/apt/sources.list && \ - echo "deb [arch=armhf,arm64] http://ports.ubuntu.com/ubuntu-ports focal-updates main restricted universe multiverse" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf,arm64] http://ports.ubuntu.com/ubuntu-ports focal-backports main restricted universe multiverse" >> /etc/apt/sources.list - -# update and upgrade system -RUN apt-get update && apt-get upgrade -qy && apt-get clean - -# install packages for buildroot -RUN apt-get install -qy qemu-user-static && \ - apt-get install -qy libmxml1:armhf libfftw3-3:armhf liblo7:armhf libsndfile1:armhf zlib1g:armhf libstdc++6:armhf && \ - apt-get install -qy libmxml1:arm64 libfftw3-3:arm64 liblo7:arm64 libsndfile1:arm64 zlib1g:arm64 libstdc++6:arm64 && \ - apt-get install -qy locales acl bash bash-completion bc curl cvs git mercurial rsync subversion sudo wget dosfstools && \ - apt-get install -qy bison bundler bzip2 cpio flex gawk gperf gzip help2man nano perl patch python tar texinfo unzip python3-dev python3-setuptools && \ - apt-get install -qy automake binutils build-essential device-tree-compiler premake4 sunxi-tools libtool-bin ncurses-dev jq && \ - apt-get install -qy libfreetype6-dev libx11-dev libxcursor-dev libxinerama-dev libxrandr-dev pkg-config && \ - apt-get clean -RUN wget http://ports.ubuntu.com/ubuntu-ports/pool/universe/p/premake/premake_3.7-1_arm64.deb && \ - dpkg -i premake_3.7-1_arm64.deb && \ - rm premake_3.7-1_arm64.deb - -RUN locale-gen en_US.UTF-8 -RUN echo "source /etc/bash_completion" >> $HOME/.bashrc - -# user configurations -ENV USER builder -ENV HOME /home/$USER - -# mod-plugin-builder settings -ENV MPB_GIT_URL https://github.com/moddevices/mod-plugin-builder - -# create user -RUN useradd -d $HOME -m -G sudo $USER -# set user to sudoers (no password required) -RUN echo "$USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/99-$USER && chmod 0440 /etc/sudoers.d/99-$USER - -# switch user -USER $USER - -# checkout mod-build-system -WORKDIR $HOME -RUN git clone $MPB_GIT_URL -WORKDIR mod-plugin-builder -RUN ./bootstrap.sh $platform && ./.clean-install.sh $platform - -# CMD -CMD ["bash"] diff --git a/docker-build b/docker-build new file mode 100755 index 00000000..778eac35 --- /dev/null +++ b/docker-build @@ -0,0 +1,28 @@ +#!/bin/sh + +PLATFORM=$1 +PACKAGE=$2 + +if [ -z "${PLATFORM}" ] || [ -z "${PACKAGE}" ] || [ ! -e "plugins-dep/configs/${PLATFORM}_defconfig" ]; then + echo "Usage: $0 " + echo " Where platform can be one of: $(echo $(ls plugins-dep/configs | grep _defconfig | sed 's/_defconfig//g' | sort))" + echo " and plugin-package-name is a folder inside ./plugins/package" + exit 1 +fi + +. ./.docker + +if ! docker image inspect ${DOCKER_IMG_PREFIX}$1 -f " " > /dev/null 2>&1; then + echo "Initializing platform $1..." + DOCKER_PLATFORMS="$1" ./docker-init + if [ $? -ne 0 ]; then + echo "Pulling the image failed! Try running \`docker compose build ${PLATFORM}\` to build one locally." + exit 1 + fi +fi + +docker compose run --rm "$1" ./build $@ + +if [ $? -eq 0 ]; then + echo "Build artifacts should be available in 'docker-workdir/$1'" +fi diff --git a/docker-build-all b/docker-build-all new file mode 100755 index 00000000..81afa87f --- /dev/null +++ b/docker-build-all @@ -0,0 +1,20 @@ +#!/bin/sh + +PACKAGE=$1 + +if [ -z "${PACKAGE}" ]; then + echo "Usage: $0 " + echo " Where plugin-package-name is a folder inside ./plugins/package" + exit 1 +fi + +. ./.docker + +for i in $DOCKER_PLATFORMS; do + ./docker-build $i $@ + + if [ $? -ne 0 ]; then + echo "Building for platform $i failed! Exiting..." + exit 1 + fi +done diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..befac1f1 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,61 @@ +version: "3.9" +services: + x86_64: + image: ${DOCKER_ORGANIZATION}/${DOCKER_IMG_PREFIX}x86_64:latest + build: + context: . + args: + MPB_PLATFORM: x86_64 + platforms: + - "linux/amd64" + - "linux/arm64" + volumes: + - type: volume + source: mpb + target: /home/builder/mod-plugin-builder + volume: + nocopy: true + - download:/home/builder/mod-workdir/download + - plugins_x86_64:/home/builder/mod-workdir/x86_64/plugins + modduo-new: + image: ${DOCKER_ORGANIZATION}/${DOCKER_IMG_PREFIX}modduo-new:latest + build: + context: . + args: + MPB_PLATFORM: modduo-new + platforms: + - "linux/amd64" + - "linux/arm64" + volumes: + - type: volume + source: mpb + target: /home/builder/mod-plugin-builder + volume: + nocopy: true + - download:/home/builder/mod-workdir/download + - plugins_modduo_new:/home/builder/mod-workdir/modduo-new/plugins +volumes: + mpb: + driver: local + driver_opts: + type: 'none' + o: 'bind' + device: '${PWD}' + download: + driver: local + driver_opts: + type: 'none' + o: 'bind' + device: '${PWD}/docker-workdir/download' + plugins_x86_64: + driver: local + driver_opts: + type: 'none' + o: 'bind' + device: '${PWD}/docker-workdir/x86_64' + plugins_modduo_new: + driver: local + driver_opts: + type: 'none' + o: 'bind' + device: '${PWD}/docker-workdir/modduo-new' diff --git a/docker-init b/docker-init new file mode 100755 index 00000000..130fa075 --- /dev/null +++ b/docker-init @@ -0,0 +1,13 @@ +#!/bin/sh + +. ./.docker + +for i in $DOCKER_PLATFORMS; do + if [ -n "$i" ] && [ -e "plugins-dep/configs/${i}_defconfig" ]; then + mkdir -p docker-workdir/$i + fi +done + +mkdir -p docker-workdir/download + +docker compose pull $DOCKER_PLATFORMS diff --git a/docker-mount.sh b/docker-mount.sh deleted file mode 100755 index 71a69766..00000000 --- a/docker-mount.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash - -set -e - -cd $(dirname ${0}) - -####################################################################################################################### -# check arguments - -PLATFORM="${1}" -DOCKER_IMAGE="${2}" - -if [ -z "${PLATFORM}" ] || [ -z "${DOCKER_IMAGE}" ] || [ ! -e "plugins-dep/configs/${PLATFORM}_defconfig" ]; then - echo "Usage: $0 " - echo " Where platform can be one of: $(echo $(ls plugins-dep/configs | grep _defconfig | sed 's/_defconfig//g' | sort))" - exit 1 -fi - -####################################################################################################################### -# Import common code and variables - -source .common - -####################################################################################################################### -# Now setup docker mountpoints (or run it if already exists) - -if docker ps -a | grep -q "mpb_${PLATFORM}"; then - docker start -i "mpb_${PLATFORM}" -else - docker run --name "mpb_${PLATFORM}" -ti -v $(pwd):/home/builder/mod-plugin-builder -v ${PLUGINS_DIR}:/home/builder/mod-workdir/${PLATFORM}/plugins "${DOCKER_IMAGE}" -fi diff --git a/docker-mpb-push-all b/docker-mpb-push-all new file mode 100755 index 00000000..45a028b3 --- /dev/null +++ b/docker-mpb-push-all @@ -0,0 +1,16 @@ +#!/bin/sh + +# You should have a builder set up consisting of at least one node building amd64 natively and one for arm64. +# Otherwise, the build would take forever and simply time out. You can use a command similar to these: +# +# docker context create --docker host=ssh://my_user@192.168.1.120 node-arm64 +# docker buildx create --append --name my_builder default +# docker buildx create --append --name my_builder node-arm64 +# docker buildx use my_builder +# + +. ./.docker + +for i in $DOCKER_PLATFORMS; do + docker buildx build --build-arg MPB_PLATFORM=$i --platform=linux/arm64,linux/amd64 -t ${DOCKER_ORGANIZATION}/${DOCKER_IMG_PREFIX}${i}:latest --push . +done diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 9d58bb0a..00000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,64 +0,0 @@ -FROM ubuntu:18.04 -LABEL maintainer="Alexandre Cunha " -ENV DEBIAN_FRONTEND noninteractive - -# platform argument must be provided, e.g. --build-arg platform=modduo -ARG platform=modduo - -# enable i386, armhf and arm64 -RUN dpkg --add-architecture i386 -RUN dpkg --add-architecture armhf -RUN dpkg --add-architecture arm64 - -RUN echo "deb [arch=amd64,i386] http://archive.ubuntu.com/ubuntu/ bionic main restricted universe multiverse" > /etc/apt/sources.list && \ - echo "deb [arch=amd64,i386] http://archive.ubuntu.com/ubuntu/ bionic-updates main restricted universe multiverse" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64,i386] http://archive.ubuntu.com/ubuntu/ bionic-backports main restricted universe multiverse" >> /etc/apt/sources.list && \ - echo "deb [arch=amd64,i386] http://security.ubuntu.com/ubuntu bionic-security main restricted universe multiverse" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf,arm64] http://ports.ubuntu.com/ubuntu-ports bionic main restricted universe multiverse" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf,arm64] http://ports.ubuntu.com/ubuntu-ports bionic-updates main restricted universe multiverse" >> /etc/apt/sources.list && \ - echo "deb [arch=armhf,arm64] http://ports.ubuntu.com/ubuntu-ports bionic-backports main restricted universe multiverse" >> /etc/apt/sources.list - -# update and upgrade system -RUN apt-get update && apt-get upgrade -qy && apt-get clean - -# NOTE multiarch-support:armhf is installed but not multiarch-support:arm64, everything still works though - -# install packages for buildroot -RUN apt-get install -qy multiarch-support:armhf qemu-user-static && \ - apt-get install -qy libmxml1:armhf libfftw3-3:armhf liblo7:armhf libsndfile1:armhf zlib1g:armhf libstdc++6:armhf && \ - apt-get install -qy libmxml1:arm64 libfftw3-3:arm64 liblo7:arm64 libsndfile1:arm64 zlib1g:arm64 libstdc++6:arm64 && \ - apt-get install -qy locales acl bash bash-completion bc curl cvs git mercurial rsync subversion sudo wget dosfstools && \ - apt-get install -qy bison bundler bzip2 cpio flex gawk gperf gzip help2man nano perl patch python tar texinfo unzip python3-dev python3-setuptools && \ - apt-get install -qy automake binutils build-essential device-tree-compiler premake4 sunxi-tools libtool-bin ncurses-dev libc6:i386 libstdc++6:i386 zlib1g:i386 jq && \ - apt-get install -qy libfreetype6-dev libx11-dev libxcursor-dev libxinerama-dev libxrandr-dev pkg-config && \ - apt-get clean -RUN wget http://archive.ubuntu.com/ubuntu/pool/universe/p/premake/premake_3.7-1_amd64.deb && \ - dpkg -i premake_3.7-1_amd64.deb && \ - rm premake_3.7-1_amd64.deb - -RUN locale-gen en_US.UTF-8 -RUN echo "source /etc/bash_completion" >> $HOME/.bashrc - -# user configurations -ENV USER builder -ENV HOME /home/$USER - -# mod-plugin-builder settings -ENV MPB_GIT_URL https://github.com/moddevices/mod-plugin-builder - -# create user -RUN useradd -d $HOME -m -G sudo $USER -# set user to sudoers (no password required) -RUN echo "$USER ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/99-$USER && chmod 0440 /etc/sudoers.d/99-$USER - -# switch user -USER $USER - -# checkout mod-build-system -WORKDIR $HOME -RUN git clone $MPB_GIT_URL -WORKDIR mod-plugin-builder -RUN ./bootstrap.sh $platform && ./.clean-install.sh $platform - -# CMD -CMD ["bash"]