Run redis integration tests in Docker network

The queue_controller server has been replaced by a separate container
that can run inside the network.

Closes replicate#262
Closes replicate#263

Signed-off-by: Ben Firshman <[email protected]>

Author: bfirsh

test-integration/test_integration/conftest.py

@@ -1,25 +1,76 @@
+from pathlib import Path
 import subprocess
 import pytest
 
-from .util import random_string, find_free_port, docker_run, get_local_ip, wait_for_port
+import redis
+
+from .util import random_string, find_free_port, docker_run, wait_for_port
 
 
 @pytest.fixture
-def redis_port():
-    container_name = "cog-test-redis-" + random_string(10)
+def docker_network():
+    name = "cog-test-" + random_string(10)
+    subprocess.run(["docker", "network", "create", name])
+    yield name
+    subprocess.run(["docker", "network", "rm", name])
+
+
+@pytest.fixture
+def redis_port(docker_network):
+    """Start a redis server inside the Docker network.
+    Inside the network, it is available at redis:6379
+    Outside the network, it is available at localhost:redis_port
+    """
     port = find_free_port()
     with docker_run(
         "redis",
-        name=container_name,
+        net_alias="redis",
+        network=docker_network,
         publish=[{"host": port, "container": 6379}],
         detach=True,
     ):
-        wait_for_port(get_local_ip(), port)
+        wait_for_port("localhost", port)
         yield port
 
 
+@pytest.fixture
+def redis_client(redis_port):
+    yield redis.Redis("localhost", redis_port)
+
+
 @pytest.fixture
 def docker_image():
     image = "cog-test-" + random_string(10)
     yield image
     subprocess.run(["docker", "rmi", "-f", image], check=False)
+
+
+@pytest.fixture
+def upload_server_image():
+    """
+    Build the upload server once for the test run. The image doesn't change.
+    """
+    subprocess.run(
+        ["docker", "build", "-t", "cog-test-upload-server", "."],
+        cwd=Path(__file__).parent.parent / "upload_server",
+        check=True,
+    )
+    return "cog-test-upload-server"
+
+
+@pytest.fixture
+def upload_server(docker_network, upload_server_image, tmpdir_factory):
+    """
+    Run a server that can be used to upload and download files from.
+
+    It is accessible at http://upload-server:5000 inside the network. The thing returned is the path for uploads.
+    """
+    tmpdir = tmpdir_factory.mktemp("uploads")
+    with docker_run(
+        upload_server_image,
+        net_alias="upload-server",
+        network=docker_network,
+        volumes=["-v", f"{tmpdir}:/uploads"],
+        detach=True,
+    ):
+        yield tmpdir

test-integration/test_integration/fixtures/file-project/predict.py

@@ -12,9 +12,7 @@ def setup(self):
     def predict(self, text, path):
         with open(path) as f:
             output = self.foo + text + f.read()
-        tmp = tempfile.NamedTemporaryFile(suffix=".txt")
-        tmp.close()
-        tmp_path = Path(tmp.name)
-        with tmp_path.open("w") as f:
-            f.write(output)
-            return tmp_path
+        tmpdir = Path(tempfile.mkdtemp())
+        with open(tmpdir / "output.txt", "w") as fh:
+            fh.write(output)
+        return tmpdir / "output.txt"