Add apt/pypi/npm mirror supports

Author: wuhuizuo

create_web_container.sh

@@ -15,7 +15,7 @@
 
 DOCKER_YAML=js/docker/docker-compose-build.yaml
 PASSWDS="$USER,hello"
-
+COMPOSE_BUILD_ARGS=""
 # Fancy colors in the terminal
 if [ -t 1 ]; then
     RED=$(tput setaf 1)
@@ -51,6 +51,9 @@ help() {
        -s        start the container after creation.
        -p        list of username password pairs.  Defaults to: [${PASSWDS}]
        -i        install systemd service, with definition in /opt/emulator
+       -n        npm mirror registry.
+       -P        python package mirror index.
+       -A        deb apt source mirror.
 EOF
     exit 1
 }
@@ -73,18 +76,20 @@ generate_keys() {
     fi
 }
 
-while getopts 'hasip:' flag; do
+while getopts 'hasip:n:P:A:' flag; do
     case "${flag}" in
     a) DOCKER_YAML="${DOCKER_YAML} -f js/docker/development.yaml" ;;
     p) PASSWDS="${OPTARG}" ;;
     h) help ;;
     s) START='yes' ;;
     i) INSTALL='yes' ;;
+    n) COMPOSE_BUILD_ARGS="${COMPOSE_BUILD_ARGS} --build-arg npm_mirror=${OPTARG}" ;;
+    P) COMPOSE_BUILD_ARGS="${COMPOSE_BUILD_ARGS} --build-arg pip_index=${OPTARG}" ;;
+    A) COMPOSE_BUILD_ARGS="${COMPOSE_BUILD_ARGS} --build-arg apt_repo_mirror=${OPTARG}" ;;
     *) help ;;
     esac
 done
 
-
 # Create the javascript protobufs
 make -C js deps
 
@@ -105,7 +110,7 @@ cp ~/.android/adbkey js/docker/certs
 
 # compose the container
 pip install docker-compose >/dev/null
-docker-compose -f ${DOCKER_YAML} build
+docker-compose -f ${DOCKER_YAML} build ${COMPOSE_BUILD_ARGS}
 rm js/docker/certs/adbkey
 
 if [ "${START}" = "yes" ]; then

emu/cloud_build.py

@@ -94,7 +94,7 @@ def cloud_build(args):
             steps.append(create_build_step(system_container, args.dest))
         else:
             for metrics in [True, False]:
-                emulator_container = EmulatorContainer(emu, system_container, args.repo, metrics)
+                emulator_container = EmulatorContainer(emu, system_container, args.repo, metrics, args.aptmirror)
                 emulators.add(emulator_container.props["emu_build_id"])
                 steps.append(create_build_step(emulator_container, args.dest))
                 images.append(emulator_container.full_name())

emu/containers/emulator_container.py

@@ -30,7 +30,7 @@ class EmulatorContainer(DockerContainer):
         """
     NO_METRICS_MESSAGE = "No metrics are collected when running this container."
 
-    def __init__(self, emulator, system_image_container, repository=None, metrics=False, extra=""):
+    def __init__(self, emulator, system_image_container, repository=None, metrics=False, extra="", apt_mirror=""):
         self.emulator_zip = AndroidReleaseZip(emulator)
         self.system_image_container = system_image_container
         self.metrics = metrics
@@ -51,6 +51,7 @@ def __init__(self, emulator, system_image_container, repository=None, metrics=Fa
         self.props["metrics"] = metrics_msg
         self.props["emu_build_id"] = self.emulator_zip.build_id()
         self.props["from_base_img"] = system_image_container.full_name()
+        self.props["run_apt_repo_replace"] = apt_mirror
 
         for expect in [
             "ro.build.version.sdk",

emu/emu_docker.py

@@ -95,7 +95,7 @@ def create_docker_image(args):
         if args.sys:
             continue
 
-        emu_docker = EmulatorContainer(emu, sys_docker, args.repo, cfg.collect_metrics(), args.extra)
+        emu_docker = EmulatorContainer(emu, sys_docker, args.repo, cfg.collect_metrics(), args.extra, args.aptmirror)
         emu_docker.build(args.dest)
 
         if args.start:
@@ -129,7 +129,7 @@ def create_docker_image_interactive(args):
     if not sys_docker.available() and not sys_docker.can_pull():
         sys_docker.build(args.dest)
 
-    emu_docker = EmulatorContainer(emu_zip, sys_docker, args.repo, metrics)
+    emu_docker = EmulatorContainer(emu_zip, sys_docker, args.repo, metrics, args.aptmirror)
     emu_docker.build(args.dest)
 
     if args.start:
@@ -220,6 +220,12 @@ def main():
         "All exposed ports are forwarded, and your private adbkey (if available) is injected but not stored.",
     )
     create_parser.add_argument("--sys", action="store_true", help="Process system image layer only.")
+    create_parser.add_argument(
+        "--aptmirror",
+        default="",
+        help="Apt repo mirror for apt installing."
+        'For example http://ftp2.cn.debian.org/',
+    )
     create_parser.set_defaults(func=create_docker_image)
 
     create_inter = subparsers.add_parser(
@@ -249,6 +255,12 @@ def main():
         action="store_true",
         help="Display arm images. Note that arm images are not hardware accelerated and are *extremely* slow.",
     )
+    create_inter.add_argument(
+        "--aptmirror",
+        default="",
+        help="Apt repo mirror for apt installing. "
+        'For example http://ftp2.cn.debian.org/',
+    )
     create_inter.set_defaults(func=create_docker_image_interactive)
 
     dist_parser = subparsers.add_parser(
@@ -268,6 +280,12 @@ def main():
     dist_parser.add_argument(
         "--sys", action="store_true", help="Write system image steps, otherwise write emulator steps."
     )
+    dist_parser.add_argument(
+        "--aptmirror",
+        default="",
+        help="Apt repo mirror for apt installing."
+        'For example http://ftp2.cn.debian.org/',
+    )
     dist_parser.add_argument(
         "emuzip",
         help="Zipfile containing the a publicly released emulator, or (canary|stable|[0-9]+) to use the latest canary, stable, or build id of the emulator to use. "

emu/templates/Dockerfile

@@ -13,16 +13,18 @@
 # limitations under the License.
 {{from_base_img}}
 
+{{run_apt_repo_replace}}
+
 # Install all the required emulator dependencies.
 # You can get these by running ./android/scripts/unix/run_tests.sh --verbose --verbose --debs | grep apt | sort -u
 # pulse audio is needed due to some webrtc dependencies.
 RUN apt-get update && apt-get install -y --no-install-recommends \
-# Emulator & video bridge dependencies
+    # Emulator & video bridge dependencies
     libc6 libdbus-1-3 libfontconfig1 libgcc1 \
     libpulse0 libtinfo5 libx11-6 libxcb1 libxdamage1 \
     libnss3 libxcomposite1 libxcursor1 libxi6 \
     libxext6 libxfixes3 zlib1g libgl1 pulseaudio socat \
-# Enable turncfg through usage of curl
+    # Enable turncfg through usage of curl
     curl ca-certificates && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
@@ -61,11 +63,11 @@ CMD ["/android/sdk/launch-emulator.sh"]
 
 # Note we should use gRPC status endpoint to check for health once the canary release is out.
 HEALTHCHECK --interval=30s \
-            --timeout=30s \
-            --start-period=30s \
-            --retries=3 \
-            CMD /android/sdk/platform-tools/adb shell getprop dev.bootcomplete | grep "1"
+    --timeout=30s \
+    --start-period=30s \
+    --retries=3 \
+    CMD /android/sdk/platform-tools/adb shell getprop dev.bootcomplete | grep "1"
 
 LABEL maintainer="{{user}}" \
-      com.google.android.emulator.description="Pixel 2 Emulator, running API {{api}}" \
-      com.google.android.emulator.version="{{tag}}-{{api}}-{{abi}}/{{emu_build_id}}"
+    com.google.android.emulator.description="Pixel 2 Emulator, running API {{api}}" \
+    com.google.android.emulator.version="{{tag}}-{{api}}-{{abi}}/{{emu_build_id}}"

emu/templates/Dockerfile.emulator

@@ -13,16 +13,18 @@
 # limitations under the License.
 FROM {{from_base_img}} AS emulator
 
+{{run_apt_repo_replace}}
+
 # Install all the required emulator dependencies.
 # You can get these by running ./android/scripts/unix/run_tests.sh --verbose --verbose --debs | grep apt | sort -u
 # pulse audio is needed due to some webrtc dependencies.
 RUN apt-get update && apt-get install -y --no-install-recommends \
-# Emulator & video bridge dependencies
+    # Emulator & video bridge dependencies
     libc6 libdbus-1-3 libfontconfig1 libgcc1 \
     libpulse0 libtinfo5 libx11-6 libxcb1 libxdamage1 \
     libnss3 libxcomposite1 libxcursor1 libxi6 \
     libxext6 libxfixes3 zlib1g libgl1 pulseaudio socat \
-# Enable turncfg through usage of curl
+    # Enable turncfg through usage of curl
     curl ca-certificates && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
@@ -61,10 +63,10 @@ CMD ["/android/sdk/launch-emulator.sh"]
 
 # Note we should use gRPC status endpoint to check for health once the canary release is out.
 HEALTHCHECK --interval=30s \
-            --timeout=30s \
-            --start-period=30s \
-            --retries=3 \
-            CMD /android/sdk/platform-tools/adb shell getprop dev.bootcomplete | grep "1"
+    --timeout=30s \
+    --start-period=30s \
+    --retries=3 \
+    CMD /android/sdk/platform-tools/adb shell getprop dev.bootcomplete | grep "1"
 
 LABEL maintainer="{{user}}" \
-      com.google.android.emulator.version="{{emu_build_id}}"
+    com.google.android.emulator.version="{{emu_build_id}}"

js/docker/nginx.Dockerfile

@@ -2,6 +2,10 @@
 FROM tiangolo/node-frontend:10 as build-stage
 WORKDIR /app
 COPY package*.json /app/
+ARG npm_mirror
+RUN if [ x"${npm_mirror}" != "x" ]; then \
+    npm config set registry ${npm_mirror}; \
+    fi
 RUN npm install
 COPY ./ /app/
 ARG configuration=production

js/jwt-provider/Dockerfile

@@ -12,10 +12,21 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 FROM debian:stretch-slim
-RUN apt-get update -y
-RUN apt-get install -y python-pip python-dev build-essential
+ARG apt_repo_mirror
+RUN if [ x"$apt_repo_mirror" != "x" ]; then \
+    sed -i -E "s#http(s)?://[^\/]+#${apt_repo_mirror}#g" /etc/apt/sources.list;  \
+    fi
+RUN apt-get update -y && \
+    apt-get install -y python-pip python-dev build-essential && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 COPY . /app
 WORKDIR /app
-RUN pip install -r requirements.txt
+ARG pip_index
+RUN if [ x"$pip_index" = "x" ]; then \
+    pip install -r requirements.txt; \
+    else \
+    pip install -i ${pip_index} -r requirements.txt; \
+    fi
 ENTRYPOINT ["python"]
 CMD ["jwt-provider.py"]