From 575d36e513f1fb75ad897338e85037ecffe28b83 Mon Sep 17 00:00:00 2001 From: Eugene Brodsky Date: Fri, 11 Nov 2022 22:53:33 -0500 Subject: [PATCH 1/7] add docker build optimized for size; do not copy models to image useful for cloud deployments. attempts to utilize docker layer caching as effectively as possible. also some quick tools to help with building --- .dockerignore | 24 ++++++++++++++- docker-build/Dockerfile.cloud | 51 +++++++++++++++++++++++++++++++ docker-build/Makefile | 56 +++++++++++++++++++++++++++++++++++ 3 files changed, 130 insertions(+), 1 deletion(-) create mode 100644 docker-build/Dockerfile.cloud create mode 100644 docker-build/Makefile diff --git a/.dockerignore b/.dockerignore index 8a0ebc50696..6b37273129b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,25 @@ * -!environment*.yml + +!backend +!frontend +!scripts +!server +!static +!LICENSE* +!main.py +!setup.py !docker-build +!ldm + +# Guard against pulling in any models that might exist in the directory tree +**/*.pt* + +# unignore configs, but only ignore the custom models.yaml, in case it exists +!configs +configs/models.yaml + +# unignore environment dirs/files, but ignore the environment.yml file or symlink in case it exists +!environment* +environment.yml + +**/__pycache__ \ No newline at end of file diff --git a/docker-build/Dockerfile.cloud b/docker-build/Dockerfile.cloud new file mode 100644 index 00000000000..c0f24d8c2c9 --- /dev/null +++ b/docker-build/Dockerfile.cloud @@ -0,0 +1,51 @@ +# Copyright (c) 2022 Eugene Brodsky (https://github.com/ebr) + +FROM nvidia/cuda:11.7.1-runtime-ubuntu22.04 AS base + +ENV DEBIAN_FRONTEND=noninteractive +# # no __pycache__ - unclear if there is a benefit +# ENV PYTHONDONTWRITEBYTECODE=1 +# unbuffered output, ensures stdout and stderr are printed in the correct order +ENV PYTHONUNBUFFERED=1 + +RUN apt update && apt install -y \ + git \ + curl \ + ncdu \ + iotop \ + bzip2 \ + libglib2.0-0 \ + libgl1-mesa-glx \ + && apt-get clean + +# Micromamba is a minimal conda implementation +ENV MAMBA_ROOT_PREFIX=/opt/conda +RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba + +WORKDIR /invokeai + +### Cache the dependencies first +# Avoid re-downloading the dependencies when unrelated files change in context +# +# We could use relative paths in the environment file, but it's not currently set up that way. +# So we copy it to the working directory to maintain compatibility with other installation methods +COPY environments-and-requirements/environment-lin-cuda.yml environment.yml + +# Patch the env file to remove installation of local package +RUN sed -i '/-e \./d' environment.yml +RUN micromamba create -y -f environment.yml &&\ + micromamba clean --all -f -y &&\ + rm -rf ${MAMBA_ROOT_PREFIX}/pkgs + +### Copy the rest of the context and install local package +COPY . . +RUN micromamba -n invokeai run pip install -e . + +### Default model config +RUN cp configs/models.yaml.example configs/models.yaml + +ENTRYPOINT ["bash"] + +EXPOSE 9090 + +CMD [ "-c", "micromamba -r ${MAMBA_ROOT_PREFIX} -n invokeai run python scripts/invoke.py --web --host 0.0.0.0"] diff --git a/docker-build/Makefile b/docker-build/Makefile new file mode 100644 index 00000000000..3fc46c31400 --- /dev/null +++ b/docker-build/Makefile @@ -0,0 +1,56 @@ +# Copyright (c) 2022 Eugene Brodsky (https://github.com/ebr) + +# InvokeAI root directory in the container +INVOKEAI_ROOT=/invokeai +# Destination directory for cache on the host +INVOKEAI_CACHEDIR=${HOME}/invokeai + +DOCKER_BUILDKIT=1 +IMAGE=local/invokeai:latest + +# Downloads will end up in ${INVOKEAI_CACHEDIR}. +# Contents can be moved to a persistent storage and used to rehydrate the cache, +# to be mounted into the container. + +# TBD: cache dir may be modified at runtime due to checksum calculations +# for custom models. Ideally the checksums would be precomputed and stored in the cache. +# Also CLI requires RW on first run when populating torch cache. + +build: + docker buildx build -t local/invokeai:latest -f Dockerfile.cloud .. + +# Populate the cache. Config is copied first, so it's there for the model preload step. +load-models: _copy-config + docker run --rm -it --runtime=nvidia --gpus=all \ + -v ${INVOKEAI_CACHEDIR}:${INVOKEAI_ROOT} \ + -v ${INVOKEAI_CACHEDIR}/root-cache:/root/.cache \ + ${IMAGE} \ + -c "micromamba -r /opt/conda -n invokeai run python scripts/load_models.py" + +run: + docker run --rm -it --runtime=nvidia --gpus=all \ + -v ${INVOKEAI_CACHEDIR}:${INVOKEAI_ROOT} \ + -v ${INVOKEAI_CACHEDIR}/root-cache:/root/.cache \ + --entrypoint bash -p9090:9090 \ + ${IMAGE} \ + -c "micromamba -r /opt/conda -n invokeai run python scripts/invoke.py --web --host 0.0.0.0" + +shell: + docker run --rm -it --runtime=nvidia --gpus=all \ + -v ${INVOKEAI_CACHEDIR}:${INVOKEAI_ROOT} \ + -v ${INVOKEAI_CACHEDIR}/root-cache:/root/.cache \ + --entrypoint bash \ + ${IMAGE} -- + +# This is an intermediate task that copies the contents of the config dir into the cache. +# This prepares the cached config dir, so that when we run the model preload with the config mounted, +# it is already populated. +_copy-config: + docker run --rm -it \ + -v ${INVOKEAI_CACHEDIR}/config:${INVOKEAI_ROOT}/tmp/config \ + --workdir ${INVOKEAI_ROOT} \ + --entrypoint bash ${IMAGE} \ + -c "cp -r ./tmp/config/* ./config/" + + +.PHONY: build preload run shell _copy-config \ No newline at end of file From 21fae59874c70f3dacd4dfb06b254927bce70f90 Mon Sep 17 00:00:00 2001 From: Eugene Brodsky Date: Sun, 13 Nov 2022 00:54:34 -0500 Subject: [PATCH 2/7] add workflow to build cloud img in ci --- .github/workflows/build-cloud-img.yml | 30 +++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/workflows/build-cloud-img.yml diff --git a/.github/workflows/build-cloud-img.yml b/.github/workflows/build-cloud-img.yml new file mode 100644 index 00000000000..5922b278109 --- /dev/null +++ b/.github/workflows/build-cloud-img.yml @@ -0,0 +1,30 @@ +name: Build cloud image +on: + push: + # TEMP until PR is ready + branches: + - docker-min + +jobs: + docker: + strategy: + fail-fast: false + matrix: + # only x86_64 for now. aarch64+cuda isn't really a thing yet + arch: + - x86_64 + runs-on: ubuntu-latest + name: ${{ matrix.arch }} + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + - name: Build container + uses: docker/build-push-action@v3 + with: + context: . + file: docker-build/Dockerfile.cloud + platforms: Linux/${{ matrix.arch }} + push: false + tags: local/invokeai:${{ matrix.arch }} From 0270b255937d5a168c6dd2deac3d0113062aa800 Mon Sep 17 00:00:00 2001 From: Eugene Brodsky Date: Tue, 15 Nov 2022 01:23:18 -0500 Subject: [PATCH 3/7] push cloud image in addition to building --- .github/workflows/build-cloud-img.yml | 40 +++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-cloud-img.yml b/.github/workflows/build-cloud-img.yml index 5922b278109..631dbf0423a 100644 --- a/.github/workflows/build-cloud-img.yml +++ b/.github/workflows/build-cloud-img.yml @@ -1,9 +1,22 @@ -name: Build cloud image +name: Build and push cloud image on: + workflow_dispatch: push: - # TEMP until PR is ready branches: + - main + - development + ## temp - docker-min + tags: + - v* + +permissions: + contents: read + packages: write + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} jobs: docker: @@ -18,13 +31,30 @@ jobs: steps: - name: Checkout uses: actions/checkout@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v4 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - - name: Build container + + - if: github.event_name != 'pull_request' + name: Docker login + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push cloud image uses: docker/build-push-action@v3 with: context: . file: docker-build/Dockerfile.cloud platforms: Linux/${{ matrix.arch }} - push: false - tags: local/invokeai:${{ matrix.arch }} + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file From ff21ed0b8aa870fd72f5b6ba1469b9e5faef9813 Mon Sep 17 00:00:00 2001 From: Eugene Brodsky Date: Sun, 20 Nov 2022 23:35:33 -0500 Subject: [PATCH 4/7] (ci) also tag docker images with git SHA --- .github/workflows/build-cloud-img.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-cloud-img.yml b/.github/workflows/build-cloud-img.yml index 631dbf0423a..b35dcc5777b 100644 --- a/.github/workflows/build-cloud-img.yml +++ b/.github/workflows/build-cloud-img.yml @@ -37,6 +37,11 @@ jobs: uses: docker/metadata-action@v4 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=sha - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 From 91c09e7b7713ab95b38e1a9fa83690d550e909f9 Mon Sep 17 00:00:00 2001 From: Eugene Brodsky Date: Mon, 21 Nov 2022 01:40:13 -0500 Subject: [PATCH 5/7] (docker) rework Makefile for easy cache population and local use --- docker-build/Makefile | 62 ++++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 33 deletions(-) diff --git a/docker-build/Makefile b/docker-build/Makefile index 3fc46c31400..a3b3c672579 100644 --- a/docker-build/Makefile +++ b/docker-build/Makefile @@ -1,56 +1,52 @@ # Copyright (c) 2022 Eugene Brodsky (https://github.com/ebr) -# InvokeAI root directory in the container -INVOKEAI_ROOT=/invokeai -# Destination directory for cache on the host +# Directory in the container where the INVOKEAI_ROOT will be mounted +INVOKEAI_ROOT=/mnt/invokeai +# Host directory to contain the model cache. Will be mounted at INVOKEAI_ROOT path in the container INVOKEAI_CACHEDIR=${HOME}/invokeai DOCKER_BUILDKIT=1 IMAGE=local/invokeai:latest -# Downloads will end up in ${INVOKEAI_CACHEDIR}. -# Contents can be moved to a persistent storage and used to rehydrate the cache, -# to be mounted into the container. +USER=$(shell id -u) +GROUP=$(shell id -g) -# TBD: cache dir may be modified at runtime due to checksum calculations -# for custom models. Ideally the checksums would be precomputed and stored in the cache. -# Also CLI requires RW on first run when populating torch cache. +# All downloaded models and config will end up in ${INVOKEAI_CACHEDIR}. +# Contents can be moved to a persistent storage and used to rehydrate the cache on another host build: docker buildx build -t local/invokeai:latest -f Dockerfile.cloud .. -# Populate the cache. Config is copied first, so it's there for the model preload step. -load-models: _copy-config +# Populate the cache. +# First, pre-seed the config dir on the host with the content from the image, +# such that the model preload step can run with the config mounted and pre-populated. +# Then, run `load-models` to cache models, VAE, other static data. +load-models: + docker run --rm -it \ + -v ${INVOKEAI_CACHEDIR}/configs:/mnt/configs \ + --entrypoint bash ${IMAGE} \ + -c "cp -r ./configs/* /mnt/configs/" docker run --rm -it --runtime=nvidia --gpus=all \ -v ${INVOKEAI_CACHEDIR}:${INVOKEAI_ROOT} \ - -v ${INVOKEAI_CACHEDIR}/root-cache:/root/.cache \ - ${IMAGE} \ - -c "micromamba -r /opt/conda -n invokeai run python scripts/load_models.py" + -v ${INVOKEAI_CACHEDIR}/.cache:/root/.cache \ + --entrypoint bash ${IMAGE} \ + -c "micromamba -n invokeai run python scripts/load_models.py --root ${INVOKEAI_ROOT}" + sudo chown -R ${USER}:${GROUP} ${INVOKEAI_CACHEDIR} +# Run the container with the cache mounted and the web server exposed on port 9090 run: docker run --rm -it --runtime=nvidia --gpus=all \ -v ${INVOKEAI_CACHEDIR}:${INVOKEAI_ROOT} \ - -v ${INVOKEAI_CACHEDIR}/root-cache:/root/.cache \ - --entrypoint bash -p9090:9090 \ - ${IMAGE} \ - -c "micromamba -r /opt/conda -n invokeai run python scripts/invoke.py --web --host 0.0.0.0" + -v ${INVOKEAI_CACHEDIR}/.cache:/root/.cache \ + --entrypoint bash -p9090:9090 ${IMAGE} \ + -c "micromamba -n invokeai run python scripts/invoke.py --web --host 0.0.0.0 --root ${INVOKEAI_ROOT}" +# Run the container with the cache mounted and open a bash shell instead of the Invoke CLI or webserver shell: docker run --rm -it --runtime=nvidia --gpus=all \ -v ${INVOKEAI_CACHEDIR}:${INVOKEAI_ROOT} \ - -v ${INVOKEAI_CACHEDIR}/root-cache:/root/.cache \ - --entrypoint bash \ - ${IMAGE} -- - -# This is an intermediate task that copies the contents of the config dir into the cache. -# This prepares the cached config dir, so that when we run the model preload with the config mounted, -# it is already populated. -_copy-config: - docker run --rm -it \ - -v ${INVOKEAI_CACHEDIR}/config:${INVOKEAI_ROOT}/tmp/config \ - --workdir ${INVOKEAI_ROOT} \ - --entrypoint bash ${IMAGE} \ - -c "cp -r ./tmp/config/* ./config/" - + -v ${INVOKEAI_CACHEDIR}/.cache:/root/.cache \ + -e INVOKEAI_ROOT=${INVOKEAI_ROOT} \ + --entrypoint bash ${IMAGE} -- -.PHONY: build preload run shell _copy-config \ No newline at end of file +.PHONY: build preload run shell \ No newline at end of file From cf3d95fbdc2452f50e4e9a58a7445396133c4e5e Mon Sep 17 00:00:00 2001 From: Eugene Brodsky Date: Tue, 22 Nov 2022 22:07:30 -0500 Subject: [PATCH 6/7] installer dir is needed during docker build --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index 6b37273129b..60b413dc044 100644 --- a/.dockerignore +++ b/.dockerignore @@ -10,6 +10,7 @@ !setup.py !docker-build !ldm +!installer # Guard against pulling in any models that might exist in the directory tree **/*.pt* From a665dba5ffa989d88b1848b17a8e57de5d0e45ff Mon Sep 17 00:00:00 2001 From: Eugene Brodsky Date: Tue, 22 Nov 2022 22:42:23 -0500 Subject: [PATCH 7/7] support the new conda-less install; further optimize docker build --- .dockerignore | 1 + docker-build/Dockerfile.cloud | 71 ++++++++++++++++++++--------------- docker-build/Makefile | 28 ++++++++------ 3 files changed, 59 insertions(+), 41 deletions(-) diff --git a/.dockerignore b/.dockerignore index 60b413dc044..a94057f6f48 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,6 +9,7 @@ !main.py !setup.py !docker-build +docker-build/Dockerfile !ldm !installer diff --git a/docker-build/Dockerfile.cloud b/docker-build/Dockerfile.cloud index c0f24d8c2c9..bc4fbae1731 100644 --- a/docker-build/Dockerfile.cloud +++ b/docker-build/Dockerfile.cloud @@ -1,13 +1,42 @@ # Copyright (c) 2022 Eugene Brodsky (https://github.com/ebr) -FROM nvidia/cuda:11.7.1-runtime-ubuntu22.04 AS base +#### Builder stage #### + +FROM library/ubuntu:22.04 AS builder ENV DEBIAN_FRONTEND=noninteractive -# # no __pycache__ - unclear if there is a benefit -# ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONDONTWRITEBYTECODE=1 # unbuffered output, ensures stdout and stderr are printed in the correct order ENV PYTHONUNBUFFERED=1 +RUN --mount=type=cache,target=/var/cache/apt \ + apt update && apt install -y \ + libglib2.0-0 \ + libgl1-mesa-glx \ + python3-venv \ + python3-pip + + +ARG APP_ROOT=/invokeai +WORKDIR ${APP_ROOT} + +ENV VIRTUAL_ENV=${APP_ROOT}/.venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +COPY . . +RUN --mount=type=cache,target=/root/.cache/pip \ + cp installer/py3.10-linux-x86_64-cuda-reqs.txt requirements.txt && \ + python3 -m venv ${VIRTUAL_ENV} &&\ + pip install --extra-index-url https://download.pytorch.org/whl/cu116 \ + torch==1.12.0+cu116 \ + torchvision==0.13.0+cu116 &&\ + pip install -r requirements.txt &&\ + pip install -e . + + +#### Runtime stage #### + +FROM ubuntu:22.04 as runtime RUN apt update && apt install -y \ git \ curl \ @@ -16,36 +45,18 @@ RUN apt update && apt install -y \ bzip2 \ libglib2.0-0 \ libgl1-mesa-glx \ + python3-venv \ + python3-pip \ && apt-get clean -# Micromamba is a minimal conda implementation -ENV MAMBA_ROOT_PREFIX=/opt/conda -RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/latest | tar -xvj bin/micromamba - -WORKDIR /invokeai - -### Cache the dependencies first -# Avoid re-downloading the dependencies when unrelated files change in context -# -# We could use relative paths in the environment file, but it's not currently set up that way. -# So we copy it to the working directory to maintain compatibility with other installation methods -COPY environments-and-requirements/environment-lin-cuda.yml environment.yml - -# Patch the env file to remove installation of local package -RUN sed -i '/-e \./d' environment.yml -RUN micromamba create -y -f environment.yml &&\ - micromamba clean --all -f -y &&\ - rm -rf ${MAMBA_ROOT_PREFIX}/pkgs - -### Copy the rest of the context and install local package -COPY . . -RUN micromamba -n invokeai run pip install -e . +ARG APP_ROOT=/invokeai +WORKDIR ${APP_ROOT} -### Default model config -RUN cp configs/models.yaml.example configs/models.yaml +ENV VIRTUAL_ENV=${APP_ROOT}/.venv +ENV PATH="$VIRTUAL_ENV/bin:$PATH" -ENTRYPOINT ["bash"] +COPY --from=builder ${APP_ROOT} ${APP_ROOT} -EXPOSE 9090 +ENTRYPOINT ["bash", "-c", "python3 scripts/invoke.py" ] -CMD [ "-c", "micromamba -r ${MAMBA_ROOT_PREFIX} -n invokeai run python scripts/invoke.py --web --host 0.0.0.0"] +CMD ["--web", "--host 0.0.0.0"] diff --git a/docker-build/Makefile b/docker-build/Makefile index a3b3c672579..e8463f0057e 100644 --- a/docker-build/Makefile +++ b/docker-build/Makefile @@ -15,13 +15,12 @@ GROUP=$(shell id -g) # Contents can be moved to a persistent storage and used to rehydrate the cache on another host build: - docker buildx build -t local/invokeai:latest -f Dockerfile.cloud .. + docker build -t local/invokeai:latest -f Dockerfile.cloud .. -# Populate the cache. -# First, pre-seed the config dir on the host with the content from the image, -# such that the model preload step can run with the config mounted and pre-populated. -# Then, run `load-models` to cache models, VAE, other static data. -load-models: +# Copy only the content of config dir first, such that the configuration +# script can run with the expected config dir already populated. +# Then, run the configuration script. +configure: docker run --rm -it \ -v ${INVOKEAI_CACHEDIR}/configs:/mnt/configs \ --entrypoint bash ${IMAGE} \ @@ -29,17 +28,24 @@ load-models: docker run --rm -it --runtime=nvidia --gpus=all \ -v ${INVOKEAI_CACHEDIR}:${INVOKEAI_ROOT} \ -v ${INVOKEAI_CACHEDIR}/.cache:/root/.cache \ - --entrypoint bash ${IMAGE} \ - -c "micromamba -n invokeai run python scripts/load_models.py --root ${INVOKEAI_ROOT}" + ${IMAGE} \ + -c "scripts/configure_invokeai.py --root ${INVOKEAI_ROOT}" sudo chown -R ${USER}:${GROUP} ${INVOKEAI_CACHEDIR} # Run the container with the cache mounted and the web server exposed on port 9090 -run: +web: docker run --rm -it --runtime=nvidia --gpus=all \ -v ${INVOKEAI_CACHEDIR}:${INVOKEAI_ROOT} \ -v ${INVOKEAI_CACHEDIR}/.cache:/root/.cache \ --entrypoint bash -p9090:9090 ${IMAGE} \ - -c "micromamba -n invokeai run python scripts/invoke.py --web --host 0.0.0.0 --root ${INVOKEAI_ROOT}" + -c "scripts/invoke.py --web --host 0.0.0.0 --root ${INVOKEAI_ROOT}" + +cli: + docker run --rm -it --runtime=nvidia --gpus=all \ + -v ${INVOKEAI_CACHEDIR}:${INVOKEAI_ROOT} \ + -v ${INVOKEAI_CACHEDIR}/.cache:/root/.cache \ + --entrypoint bash ${IMAGE} \ + -c "scripts/invoke.py --root ${INVOKEAI_ROOT}" # Run the container with the cache mounted and open a bash shell instead of the Invoke CLI or webserver shell: @@ -49,4 +55,4 @@ shell: -e INVOKEAI_ROOT=${INVOKEAI_ROOT} \ --entrypoint bash ${IMAGE} -- -.PHONY: build preload run shell \ No newline at end of file +.PHONY: build configure web cli shell \ No newline at end of file