feat: use local db for integration tests

.github/workflows/ci.yaml

@@ -144,41 +144,29 @@ jobs:
       - name: Setup Backend
         uses: ./.github/actions/setup-backend
 
-      - name: Configure credentials variables
+      - name: Run test database and redis services
         run: |
-          echo -n "${{ secrets.PG_JSON }}" | base64 --decode > application_default_credentials.json
-          chmod 777 application_default_credentials.json
-          mkdir -p backend/runtime/secrets
-          echo "${{ secrets.PG_PASSWORD }}" > backend/runtime/secrets/postgres_password_secret
-          echo "DB_DEFAULT_USER=${{ secrets.PG_USERNAME }}" >> $GITHUB_ENV
-          echo "DJANGO_SECRET_KEY=$(openssl rand -base64 22)" >> $GITHUB_ENV
-          echo "DB_DEFAULT_NAME=kcidb" >> $GITHUB_ENV
-          cp .env.backend.example .env.backend
-          cp .env.db.example .env.db
-          cp .env.proxy.example .env.proxy
-
-      - name: Run database proxy and backend docker services
-        run: docker compose up backend -d --build
-
-      - name: Wait for backend to be ready
+          docker compose -f docker-compose.test.yml up test_db redis -d
+          sleep 5
+
+      - name: Execute migrations
+        run: |
+          docker compose -f docker-compose.test.yml run --rm test_backend python manage.py migrate
+
+      - name: Populate database with test data
         run: |
-          for i in {1..20}; do
-            BACKEND_STATUS=$(docker ps -q -f "name=dashboard_backend_service" -f "status=running")
-            PROXY_STATUS=$(docker ps -q -f "name=cloudsql-proxy" -f "status=running")
-
-            if [[ -n "$BACKEND_STATUS" && -n "$PROXY_STATUS" ]] then
-                echo "Backend is ready!"
-                docker ps
-                sleep 5
-                break
-            fi
-            echo "Waiting for backend to be ready... $i"
-            sleep 5
-          done
+          docker compose -f docker-compose.test.yml run --rm test_backend python manage.py seed_test_data --clear
+
+      - name: Start test backend
+        run: |
+          docker compose -f docker-compose.test.yml up test_backend -d
+          echo "Waiting for test backend to be ready..."
+          sleep 5
+          docker ps
 
       - name: Run integration tests with coverage
         run: |
-          poetry run pytest -m "integration" --run-all --cov=kernelCI_app --cov=kernelCI_cache --cov-report=term-missing
+          TEST_BASE_URL=http://localhost:8001 poetry run pytest -m "integration" --use-local-db --run-all --cov=kernelCI_app --cov=kernelCI_cache --cov-report=term-missing
           cp .coverage coverage-integration.sqlite
         working-directory: ./backend
 
@@ -189,7 +177,9 @@ jobs:
           path: backend/coverage-integration.sqlite
 
       - name: Clean containers
-        run: docker compose down --volumes --remove-orphans && docker system prune -af
+        run: |
+          docker compose -f docker-compose.test.yml down --volumes --remove-orphans
+          docker system prune -af
 
   update-coverage-badge:
     if: github.ref == format('refs/heads/{0}', github.event.repository.default_branch)

backend/Dockerfile

@@ -25,7 +25,12 @@ COPY . /backend
 COPY ./utils/docker/entrypoint.sh /entrypoint.sh
 WORKDIR /backend
 
-RUN poetry install --no-interaction --only main
+ARG INSTALL_DEV_DEPS=false
+RUN if [ "$INSTALL_DEV_DEPS" = "true" ]; then \
+        poetry install --no-interaction; \
+    else \
+        poetry install --no-interaction --only main; \
+    fi
 
 ENV DJANGO_APP="kernelCI"
 

backend/conftest.py

@@ -1,12 +1,33 @@
 import tomllib
 from pytest import Item
 import os
+import django
+from django.conf import settings
 
 
 def pytest_addoption(parser):
     parser.addoption(
         "--run-all", action="store_true", default=False, help="run all test cases"
     )
+    parser.addoption(
+        "--use-local-db",
+        action="store_true",
+        default=False,
+        help="use local test database instead of remote",
+    )
+
+
+def pytest_configure(config):
+    """Configure Django settings for tests."""
+    if not settings.configured:
+        use_local_db = config.getoption("--use-local-db")
+
+        if use_local_db:
+            os.environ.setdefault("DJANGO_SETTINGS_MODULE", "kernelCI.test_settings")
+        else:
+            os.environ.setdefault("DJANGO_SETTINGS_MODULE", "kernelCI.settings")
+
+        django.setup()
 
 
 def get_test_markers() -> list[str]:

backend/kernelCI/test_settings.py

@@ -0,0 +1,95 @@
+"""
+Test-specific Django settings for integration tests with local database.
+"""
+
+# Override database configuration for tests
+DATABASES = {
+    "default": {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": "kcidb_test",
+        "USER": "test_user",
+        "PASSWORD": "test_password",
+        "HOST": "test_db",
+        "PORT": "5432",
+        "OPTIONS": {
+            "connect_timeout": 5,
+        },
+    },
+    "dashboard_db": {
+        "ENGINE": "django.db.backends.postgresql",
+        "NAME": "dashboard_test",
+        "USER": "test_user",
+        "PASSWORD": "test_password",
+        "HOST": "test_db",
+        "PORT": "5432",
+        "OPTIONS": {
+            "connect_timeout": 5,
+        },
+    },
+    "cache": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": ":memory:",
+    },
+    "notifications": {
+        "ENGINE": "django.db.backends.sqlite3",
+        "NAME": ":memory:",
+    },
+}
+
+
+# Disable migrations for faster tests
+class DisableMigrations:
+    def __contains__(self, item):
+        return True
+
+    def __getitem__(self, item):
+        return None
+
+
+MIGRATION_MODULES = DisableMigrations()
+
+# Disable debug for tests
+DEBUG = False
+DEBUG_SQL_QUERY = False
+
+# Use in-memory cache for tests
+CACHES = {
+    "default": {
+        "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
+        "LOCATION": "test-cache",
+    }
+}
+
+# Disable CORS for tests
+CORS_ALLOW_ALL_ORIGINS = True
+
+# Shorter cache timeout for tests
+CACHE_TIMEOUT = 60
+
+# Disable security features for tests
+SESSION_COOKIE_SECURE = False
+CSRF_COOKIE_SECURE = False
+SECURE_SSL_REDIRECT = False
+SECURE_HSTS_SECONDS = 0
+
+# Disable cron jobs for tests
+CRONJOBS = []
+
+# Use test email backend
+EMAIL_BACKEND = "django.core.mail.backends.locmem.EmailBackend"
+
+# Disable logging for tests
+LOGGING = {
+    "version": 1,
+    "disable_existing_loggers": True,
+    "handlers": {
+        "null": {
+            "class": "logging.NullHandler",
+        },
+    },
+    "root": {
+        "handlers": ["null"],
+    },
+}
+
+TEST_BASE_URL = "http://localhost:8001/"