Skip to content

Commit 1c064e0

Browse files
wconstabwconstab
committed
Update (base update)
[ghstack-poisoned]
0 parents  commit 1c064e0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

79 files changed

+53676
-0
lines changed

‎.ci/docker/README.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Docker images for TorchTitan CI
2+
3+
This directory contains everything needed to build the Docker images
4+
that are used in TorchTitan CI. The content of this directory are copied
5+
from PyTorch CI https://github.com/pytorch/pytorch/tree/main/.ci/docker.
6+
It also uses the same directory structure as PyTorch.
7+
8+
## Contents
9+
10+
* `build.sh` -- dispatch script to launch all builds
11+
* `common` -- scripts used to execute individual Docker build stages
12+
* `ubuntu` -- Dockerfile for Ubuntu image for CPU build and test jobs
13+
14+
## Usage
15+
16+
```bash
17+
# Generic usage
18+
./build.sh "${IMAGE_NAME}" "${DOCKER_BUILD_PARAMETERS}"
19+
20+
# Build a specific image
21+
./build.sh torchtitan-ubuntu-20.04-clang12 -t myimage:latest
22+
```

‎.ci/docker/build.sh‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -exu
9+
10+
IMAGE_NAME="$1"
11+
shift
12+
13+
echo "Building ${IMAGE_NAME} Docker image"
14+
15+
OS=ubuntu
16+
OS_VERSION=20.04
17+
CLANG_VERSION=""
18+
PYTHON_VERSION=3.11
19+
MINICONDA_VERSION=24.3.0-0
20+
21+
case "${IMAGE_NAME}" in
22+
torchtitan-ubuntu-20.04-clang12)
23+
CLANG_VERSION=12
24+
;;
25+
*)
26+
echo "Invalid image name ${IMAGE_NAME}"
27+
exit 1
28+
esac
29+
30+
docker build \
31+
--no-cache \
32+
--progress=plain \
33+
--build-arg "OS_VERSION=${OS_VERSION}" \
34+
--build-arg "CLANG_VERSION=${CLANG_VERSION}" \
35+
--build-arg "PYTHON_VERSION=${PYTHON_VERSION}" \
36+
--build-arg "MINICONDA_VERSION=${MINICONDA_VERSION}" \
37+
-f "${OS}"/Dockerfile \
38+
"$@" \
39+
.

‎.ci/docker/common/install_base.sh‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -ex
9+
10+
install_ubuntu() {
11+
apt-get update
12+
13+
apt-get install -y --no-install-recommends \
14+
build-essential \
15+
ca-certificates \
16+
curl \
17+
git \
18+
wget \
19+
sudo \
20+
vim \
21+
jq \
22+
vim \
23+
unzip \
24+
gdb \
25+
rsync \
26+
libssl-dev \
27+
zip
28+
29+
# Cleanup package manager
30+
apt-get autoclean && apt-get clean
31+
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
32+
}
33+
34+
# Install base packages depending on the base OS
35+
ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
36+
case "$ID" in
37+
ubuntu)
38+
install_ubuntu
39+
;;
40+
*)
41+
echo "Unable to determine OS..."
42+
exit 1
43+
;;
44+
esac

‎.ci/docker/common/install_clang.sh‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -ex
9+
10+
install_ubuntu() {
11+
apt-get update
12+
13+
apt-get install -y --no-install-recommends clang-"$CLANG_VERSION"
14+
apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"
15+
# Also require LLD linker from llvm and libomp to build PyTorch from source
16+
apt-get install -y lld "libomp-${CLANG_VERSION}-dev"
17+
18+
# Use update-alternatives to make this version the default
19+
update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50
20+
update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-"$CLANG_VERSION" 50
21+
# Override cc/c++ to clang as well
22+
update-alternatives --install /usr/bin/cc cc /usr/bin/clang 50
23+
update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 50
24+
25+
# Cleanup package manager
26+
apt-get autoclean && apt-get clean
27+
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
28+
}
29+
30+
if [ -n "$CLANG_VERSION" ]; then
31+
# Install base packages depending on the base OS
32+
ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"')
33+
case "$ID" in
34+
ubuntu)
35+
install_ubuntu
36+
;;
37+
*)
38+
echo "Unable to determine OS..."
39+
exit 1
40+
;;
41+
esac
42+
fi

‎.ci/docker/common/install_conda.sh‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -ex
9+
10+
# shellcheck source=/dev/null
11+
source "$(dirname "${BASH_SOURCE[0]}")/utils.sh"
12+
13+
install_miniconda() {
14+
BASE_URL="https://repo.anaconda.com/miniconda"
15+
CONDA_FILE="Miniconda3-py${PYTHON_VERSION//./}_${MINICONDA_VERSION}-Linux-x86_64.sh"
16+
17+
mkdir -p /opt/conda
18+
chown ci-user:ci-user /opt/conda
19+
20+
pushd /tmp
21+
wget -q "${BASE_URL}/${CONDA_FILE}"
22+
# Install miniconda
23+
as_ci_user bash "${CONDA_FILE}" -b -f -p "/opt/conda"
24+
# Clean up the download file
25+
rm "${CONDA_FILE}"
26+
popd
27+
28+
sed -e 's|PATH="\(.*\)"|PATH="/opt/conda/bin:\1"|g' -i /etc/environment
29+
export PATH="/opt/conda/bin:$PATH"
30+
}
31+
32+
install_python() {
33+
pushd /opt/conda
34+
# Install the correct Python version
35+
as_ci_user conda create -n "py_${PYTHON_VERSION}" -y --file /opt/conda/conda-env-ci.txt python="${PYTHON_VERSION}"
36+
popd
37+
}
38+
39+
install_pip_dependencies() {
40+
pushd /opt/conda
41+
# Install all Python dependencies
42+
pip_install -r /opt/conda/dev-requirements.txt
43+
pip_install -r /opt/conda/requirements.txt
44+
popd
45+
}
46+
47+
fix_conda_ubuntu_libstdcxx() {
48+
cat /etc/issue
49+
# WARNING: This is a HACK from PyTorch core to be able to build PyTorch on 22.04.
50+
# Specifically, ubuntu-20+ all comes lib libstdc++ newer than 3.30+, but anaconda
51+
# is stuck with 3.29. So, remove libstdc++6.so.3.29 as installed by
52+
# https://anaconda.org/anaconda/libstdcxx-ng/files?version=11.2.0
53+
#
54+
# PyTorch sev: https://github.com/pytorch/pytorch/issues/105248
55+
# Ref: https://github.com/pytorch/pytorch/blob/main/.ci/docker/common/install_conda.sh
56+
if grep -e "2[02].04." /etc/issue >/dev/null; then
57+
rm "/opt/conda/envs/py_${PYTHON_VERSION}/lib/libstdc++.so.6"
58+
fi
59+
}
60+
61+
install_miniconda
62+
install_python
63+
install_pip_dependencies
64+
fix_conda_ubuntu_libstdcxx

‎.ci/docker/common/install_gcc.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -ex
9+
10+
if [ -n "$GCC_VERSION" ]; then
11+
12+
apt-get update
13+
apt-get install -y g++-"$GCC_VERSION"
14+
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-"$GCC_VERSION" 50
15+
update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-"$GCC_VERSION" 50
16+
update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-"$GCC_VERSION" 50
17+
18+
# Cleanup package manager
19+
apt-get autoclean && apt-get clean
20+
rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
21+
22+
fi

‎.ci/docker/common/install_user.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
set -ex
9+
10+
# Same as ec2-user
11+
echo "ci-user:x:1000:1000::/var/lib/ci-user:" >> /etc/passwd
12+
echo "ci-user:x:1000:" >> /etc/group
13+
# Needed on Focal or newer
14+
echo "ci-user:*:19110:0:99999:7:::" >> /etc/shadow
15+
16+
# Create $HOME
17+
mkdir -p /var/lib/ci-user
18+
chown ci-user:ci-user /var/lib/ci-user
19+
20+
# Allow sudo
21+
echo 'ci-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ci-user
22+
23+
# Test that sudo works
24+
sudo -u ci-user sudo -v

‎.ci/docker/common/utils.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
# Copyright (c) Meta Platforms, Inc. and affiliates.
3+
# All rights reserved.
4+
#
5+
# This source code is licensed under the BSD-style license found in the
6+
# LICENSE file in the root directory of this source tree.
7+
8+
as_ci_user() {
9+
# NB: unsetting the environment variables works around a conda bug
10+
# https://github.com/conda/conda/issues/6576
11+
# NB: Pass on PATH and LD_LIBRARY_PATH to sudo invocation
12+
# NB: This must be run from a directory that the user has access to
13+
sudo -E -H -u ci-user env -u SUDO_UID -u SUDO_GID -u SUDO_COMMAND -u SUDO_USER env "PATH=${PATH}" "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}" "$@"
14+
}
15+
16+
conda_install() {
17+
# Ensure that the install command don't upgrade/downgrade Python
18+
# This should be called as
19+
# conda_install pkg1 pkg2 ... [-c channel]
20+
as_ci_user conda install -q -n "py_${PYTHON_VERSION}" -y python="${PYTHON_VERSION}" "$@"
21+
}
22+
23+
conda_run() {
24+
as_ci_user conda run -n "py_${PYTHON_VERSION}" --no-capture-output "$@"
25+
}
26+
27+
pip_install() {
28+
as_ci_user conda run -n "py_${PYTHON_VERSION}" pip install --progress-bar off "$@"
29+
}

‎.ci/docker/conda-env-ci.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
cmake=3.22.1
2+
ninja=1.10.2

‎.ci/docker/dev-requirements.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
expecttest==0.1.6
2+
pytest==7.3.2
3+
pytest-cov
4+
pre-commit

0 commit comments

Comments
 (0)