Python 3 version fixes

This migrates the build system to poetry, and removes support for
python2.

It contains a series of small fixes:

 - Improve launcher logging
 - Remove python2 support
 - Add some typing information
 - Split build into sys and emu directories

Author: Erwin Jansen

configure.sh

@@ -18,24 +18,24 @@ if [ "${BASH_SOURCE-}" = "$0" ]; then
     exit 33
 fi
 
-PYTHON=python
+PYTHON=python3
+VENV=.venv
 
-if [ ! -f "./venv/bin/activate" ]; then
+if [ ! -f "./$VENV/bin/activate" ]; then
   # Prefer python3 if it is available.
   if command -v python3 &>/dev/null; then
      echo "Using python 3"
-     PYTHON=python3
-     $PYTHON -m venv venv
-     [ -e ./venv/bin/pip ] && ./venv/bin/pip install --upgrade pip
-     [ -e ./venv/bin/pip ] && ./venv/bin/pip install --upgrade setuptools
+     $PYTHON -m venv $VENV
+     [ -e ./$VENV/bin/pip ] && ./$VENV/bin/pip install --upgrade pip
+     [ -e ./$VENV/bin/pip ] && ./$VENV/bin/pip install --upgrade setuptools
   else
     echo "Using python 2 ----<< Deprecated! See: https://python3statement.org/.."
-    $PYTHON -m virtualenv &>/dev/null || { echo "This script relies on virtualenv, you can install it with 'pip install virtualenv' (https://virtualenv.pypa.io)"; return ; }
-    $PYTHON -m virtualenv venv
+    echo "You need to upgrade to python3"
+    exit 33
   fi
 fi
-if [ -e ./venv/bin/activate ]; then
-   . ./venv/bin/activate
-   $PYTHON setup.py develop
+if [ -e ./$VENV/bin/activate ]; then
+   . ./$VENV/bin/activate
+   pip install -e .
    echo "Ready to run emu-docker!"
 fi

emu/containers/docker_container.py

@@ -17,51 +17,11 @@
 import sys
 import shutil
 import abc
+from pathlib import Path
 
 import docker
-from tqdm import tqdm
 from emu.utils import mkdir_p
-
-
-class ProgressTracker(object):
-    """Tracks progress using tqdm for a set of layers that are pushed."""
-
-    def __init__(self):
-        # This tracks the information for a given layer id.
-        self.progress = {}
-        self.idx = -1
-
-    def __del__(self):
-        for k in self.progress:
-            self.progress[k]["tqdm"].close()
-
-    def update(self, entry):
-        """Update the progress bars given a an entry.."""
-        if "id" not in entry:
-            return
-
-        identity = entry["id"]
-        if identity not in self.progress:
-            self.idx += 1
-            self.progress[identity] = {
-                "tqdm": tqdm(total=0, position=self.idx, unit="B", unit_scale=True),  # The progress bar
-                "total": 0,  # Total of bytes we are shipping
-                "status": "",  # Status message.
-                "current": 0,  # Current of total already send.
-            }
-        prog = self.progress[identity]
-
-        total = int(entry.get("progressDetail", {}).get("total", -1))
-        current = int(entry.get("progressDetail", {}).get("current", 0))
-        if prog["total"] != total and total != -1:
-            prog["total"] = total
-            prog["tqdm"].reset(total=total)
-        if prog["status"] != entry["status"]:
-            prog["tqdm"].set_description("{0} {1}".format(entry.get("status"), identity))
-        if current != 0:
-            diff = current - prog["current"]
-            prog["current"] = current
-            prog["tqdm"].update(diff)
+from emu.containers.progress_tracker import ProgressTracker
 
 
 class DockerContainer(object):
@@ -86,25 +46,30 @@ def get_api_client(self):
             api_client = docker.APIClient()
             logging.info(api_client.version())
             return api_client
-        except:
-            logging.exception("Failed to create default client, trying domain socket.", exc_info=True)
+        except Exception as _err:
+            logging.exception(
+                "Failed to create default client, trying domain socket.", exc_info=True
+            )
 
         api_client = docker.APIClient(base_url="unix://var/run/docker.sock")
         logging.info(api_client.version())
         return api_client
 
     def push(self):
         image = self.full_name()
-        print("Pushing docker image: {}.. be patient this can take a while!".format(self.full_name()))
+        print(
+            f"Pushing docker image: {self.full_name()}.. be patient this can take a while!"
+        )
+
         tracker = ProgressTracker()
         try:
             client = docker.from_env()
             result = client.images.push(image, "latest", stream=True, decode=True)
             for entry in result:
                 tracker.update(entry)
-            self.docker_image().tag("{}{}:latest".format(self.repo, self.image_name()))
-        except:
-            logging.exception("Failed to push image.", exc_info=True)
+            self.docker_image().tag(f"{self.repo}{self.image_name()}:latest")
+        except Exception as err:
+            logging.error("Failed to push image due to %s", err, exc_info=True)
             logging.warning("You can manually push the image as follows:")
             logging.warning("docker push %s", image)
 
@@ -123,24 +88,26 @@ def launch(self, port_map):
                 detach=True,
                 ports=port_map,
             )
-            print("Launched {} (id:{})".format(container.name, container.id))
-            print("docker logs -f {}".format(container.name))
-            print("docker stop {}".format(container.name))
+            print(f"Launched {container.name} (id:{container.id})")
+            print(f"docker logs -f {container.name}")
+            print(f"docker stop {container.name}")
             return container
-        except:
-            logging.exception("Unable to run the %s", image_sha)
+        except Exception as err:
+            logging.exception("Unable to run the %s due to %s", image.id, err)
             print("Unable to start the container, try running it as:")
-            print("./run.sh ", image_sha)
+            print(f"./run.sh {image.id}")
 
-    def create_container(self, dest):
+    def create_container(self, dest: Path):
         """Creates the docker container, returning the sha of the container, or None in case of failure."""
         identity = None
         image_tag = self.full_name()
-        print("docker build {} -t {}".format(dest, image_tag))
+        print(f"docker build {dest} -t {image_tag}")
         try:
             api_client = self.get_api_client()
-            logging.info("build(path=%s, tag=%s, rm=True, decode=True)", dest, image_tag)
-            result = api_client.build(path=dest, tag=image_tag, rm=True, decode=True)
+            logging.info(
+                "build(path=%s, tag=%s, rm=True, decode=True)", dest, image_tag
+            )
+            result = api_client.build(path=str(dest.absolute()), tag=image_tag, rm=True, decode=True)
             for entry in result:
                 if "stream" in entry:
                     sys.stdout.write(entry["stream"])
@@ -149,17 +116,18 @@ def create_container(self, dest):
             client = docker.from_env()
             image = client.images.get(identity)
             image.tag(self.repo + self.image_name(), "latest")
-        except:
-            logging.exception("Failed to create container.", exc_info=True)
+        except Exception as err:
+            logging.error("Failed to create container due to %s.", err, exc_info=True)
             logging.warning("You can manually create the container as follows:")
             logging.warning("docker build -t %s %s", image_tag, dest)
 
         return identity
 
-    def clean(self, dest):
-        if os.path.exists(dest):
+    def clean(self, dest: Path):
+        if dest.exists():
             shutil.rmtree(dest)
-        mkdir_p(dest)
+
+        dest.mkdir(parents=True)
 
     def pull(self, image, tag):
         """Tries to retrieve the given image and tag.
@@ -173,22 +141,24 @@ def pull(self, image, tag):
             for entry in result:
                 tracker.update(entry)
         except:
-            logging.info("Failed to retrieve image, this is not uncommon.", exc_info=True)
+            logging.info(
+                "Failed to retrieve image, this is not uncommon.", exc_info=True
+            )
             return False
 
         return True
 
     def full_name(self):
         if self.repo:
-            return "{}{}:{}".format(self.repo, self.image_name(), self.docker_tag())
+            return f"{self.repo}{self.image_name()}:{self.docker_tag()}"
         return (self.image_name(), self.docker_tag())
 
     def latest_name(self):
         if self.repo:
-            return "{}{}:{}".format(self.repo, self.image_name(), "latest")
+            return f"{self.repo}{self.image_name()}:{self.docker_tag()}"
         return (self.image_name(), "latest")
 
-    def create_cloud_build_step(self, dest):
+    def create_cloud_build_step(self, dest: Path):
         return {
             "name": "gcr.io/cloud-builders/docker",
             "args": [
@@ -197,7 +167,7 @@ def create_cloud_build_step(self, dest):
                 self.full_name(),
                 "-t",
                 self.latest_name(),
-                os.path.basename(dest),
+                dest.absolute().parent,
             ],
         }
 
@@ -218,7 +188,7 @@ def available(self):
         """True if this container image is locally available."""
         return self.docker_image() != None
 
-    def build(self, dest):
+    def build(self, dest: Path):
         self.write(dest)
         return self.create_container(dest)
 
@@ -227,7 +197,7 @@ def can_pull(self):
         return self.pull(self.image_name(), self.docker_tag())
 
     @abc.abstractmethod
-    def write(self, destination):
+    def write(self, destination: Path):
         """Method responsible for writing the Dockerfile and all necessary files to build a container.
 
         Args:

emu/containers/progress_tracker.py

@@ -0,0 +1,60 @@
+# Copyright 2019 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from tqdm import tqdm
+
+
+class ProgressTracker:
+    """
+    A class that tracks progress using tqdm for a set of layers that are pushed.
+    """
+
+    def __init__(self):
+        """        Initializes a new instance of ProgressTracker with an empty progress dictionary.        """
+        self.progress = {}
+
+    def __del__(self):
+        """        Closes the tqdm progress bars for all entries in the progress dictionary.        """
+        for _key, value in self.progress.items():
+            value["tqdm"].close()
+
+    def update(self, entry):
+        """Updates the progress bars given an entry dictionary."""
+        if "id" not in entry:
+            return
+
+        identity = entry["id"]
+        if identity not in self.progress:
+            self.progress[identity] = {
+                "tqdm": tqdm(total=0, unit="B", unit_scale=True),  # The progress bar
+                "total": 0,  # Total of bytes we are shipping
+                "status": "",  # Status message.
+                "current": 0,  # Current of total already send.
+            }
+
+        prog = self.progress[identity]
+        total = int(entry.get("progressDetail", {}).get("total", -1))
+        current = int(entry.get("progressDetail", {}).get("current", 0))
+
+        if prog["total"] != total and total != -1:
+            prog["total"] = total
+            prog["tqdm"].reset(total=total)
+
+        if prog["status"] != entry["status"]:
+            prog["status"] = entry["status"]
+            prog["tqdm"].set_description(f"{entry.get('status')} {identity}")
+
+        if current != 0:
+            diff = current - prog["current"]
+            prog["current"] = current
+            prog["tqdm"].update(diff)

emu/containers/system_image_container.py

@@ -13,7 +13,6 @@
 # limitations under the License.
 import logging
 import os
-import shutil
 
 from emu.android_release_zip import SystemImageReleaseZip
 from emu.platform_tools import PlatformTools
@@ -40,17 +39,17 @@ def _copy_adb_to(self, dest):
         tools = PlatformTools()
         tools.extract_adb(dest)
 
-    def write(self, dest):
+    def write(self, destination):
         # We do not really want to overwrite if the files already exist.
         # Make sure the destination directory is empty.
         if self.system_image_zip is None:
-            self.system_image_zip = SystemImageReleaseZip(self.system_image_info.download(dest))
+            self.system_image_zip = SystemImageReleaseZip(self.system_image_info.download(destination))
 
-        writer = TemplateWriter(dest)
-        self._copy_adb_to(dest)
+        writer = TemplateWriter(destination)
+        self._copy_adb_to(destination)
 
         props = self.system_image_zip.props
-        dest_zip = os.path.basename(self.system_image_zip.copy(dest))
+        dest_zip = os.path.basename(self.system_image_zip.copy(destination))
         props["system_image_zip"] = dest_zip
         writer.write_template(
             "Dockerfile.system_image",