refactor: auto-scaling and observability components

.github/workflows/ci.yml

@@ -43,7 +43,7 @@ jobs:
 
   integration-tests:
     name: Integration tests
-    needs: fast-checks
+    # needs: fast-checks
     if: github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ubuntu-latest
     steps:
@@ -89,6 +89,7 @@ jobs:
           echo "=== eoAPI Deployment ==="
           export RELEASE_NAME="${RELEASE_NAME}"
           export PGO_VERSION="${{ env.PGO_VERSION }}"
+          export GITHUB_SHA="${{ github.sha }}"
           export CI_MODE=true
 
           # Deploy using consolidated script with CI mode
@@ -109,7 +110,7 @@ jobs:
         run: |
           ./scripts/debug-deployment.sh
 
-      - name: Cleanup
+      - name: Cleanup integration test
         if: always()
         run: |
           helm uninstall "$RELEASE_NAME" -n eoapi || true
@@ -158,3 +159,122 @@ jobs:
             head -1 "$file" | grep -q "^---$" || { echo "❌ Missing frontmatter: $file"; missing=1; }
           done
           exit $missing
+
+  observability-tests:
+    name: Observability tests
+    if: github.event.pull_request.head.repo.full_name == github.repository
+    permissions:
+      contents: 'read'
+      id-token: 'write'
+    # needs: integration-tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Start K3s cluster
+        uses: jupyterhub/action-k3s-helm@v4
+        with:
+          k3s-channel: latest
+          helm-version: ${{ env.HELM_VERSION }}
+          metrics-enabled: false
+          docker-enabled: true
+
+      - name: Set release name
+        run: echo "RELEASE_NAME=eoapi-$(echo "${{ github.sha }}" | cut -c1-8)" >> "$GITHUB_ENV"
+
+      - name: Wait for K3s readiness
+        run: |
+          echo "=== Waiting for K3s cluster to be ready ==="
+
+          # The action already sets up kubectl context, just verify it works
+          kubectl cluster-info
+          kubectl get nodes
+
+          # Wait for core components
+          kubectl wait --for=condition=Ready pod -l k8s-app=kube-dns -n kube-system --timeout=300s
+          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=traefik -n kube-system --timeout=300s
+
+          # Verify Traefik CRDs
+          timeout=300; counter=0
+          for crd in "middlewares.traefik.io" "ingressroutes.traefik.io"; do
+            while [ $counter -lt $timeout ] && ! kubectl get crd "$crd" &>/dev/null; do
+              sleep 3; counter=$((counter + 3))
+            done
+            [ $counter -ge $timeout ] && { echo "❌ Timeout waiting for $crd"; exit 1; }
+          done
+
+          echo "✅ K3s cluster ready"
+
+      - name: Deploy eoAPI with monitoring
+        run: |
+          echo "=== Deploying eoAPI with monitoring stack ==="
+          export RELEASE_NAME="$RELEASE_NAME"
+          export PGO_VERSION="${{ env.PGO_VERSION }}"
+          export GITHUB_SHA="${{ github.sha }}"
+          export CI_MODE=true
+          export OBSERVABILITY_MODE=true
+
+          # Deploy using consolidated script with observability mode enabled
+          ./scripts/deploy.sh --ci
+
+      - name: Wait for monitoring stack to be ready
+        run: |
+          echo "=== Waiting for monitoring components ==="
+
+          # Wait for metrics-server first (required for HPA)
+          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=metrics-server -n eoapi --timeout=300s || echo "metrics-server not ready"
+
+          # Wait for Prometheus server
+          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/component=server,app.kubernetes.io/name=prometheus -n eoapi --timeout=300s || echo "Prometheus server not ready"
+
+          # Wait for Grafana
+          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=grafana -n eoapi --timeout=300s || echo "Grafana not ready"
+
+          # Wait for prometheus-adapter
+          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/name=prometheus-adapter -n eoapi --timeout=300s || echo "prometheus-adapter not ready"
+
+          # Give time for HPA to be created and configured
+          echo "=== Waiting for HPA creation ==="
+          sleep 60
+
+          echo "=== Final monitoring stack status ==="
+          kubectl get pods -n eoapi | grep -E "(prometheus|grafana|metrics-server)" || true
+          kubectl get hpa -n eoapi || echo "No HPA resources found yet"
+
+      - name: Validate core eoAPI services
+        run: |
+          echo "=== Validating core eoAPI services ==="
+
+          # Wait for core application pods to be ready
+          kubectl wait --for=condition=Ready pod -l app="$RELEASE_NAME"-stac -n eoapi --timeout=300s
+          kubectl wait --for=condition=Ready pod -l app="$RELEASE_NAME"-raster -n eoapi --timeout=300s
+          kubectl wait --for=condition=Ready pod -l app="$RELEASE_NAME"-vector -n eoapi --timeout=300s
+
+          echo "✅ Core eoAPI services are ready"
+
+      - name: Run observability tests
+        run: |
+          echo "=== Running observability test suite ==="
+          export RELEASE_NAME="$RELEASE_NAME"
+          export NAMESPACE="eoapi"
+
+          # Install python dependencies for testing
+          python -m pip install --upgrade pip
+          pip install pytest requests psycopg2-binary
+
+          # Run observability tests
+          python -m pytest .github/workflows/tests/test_observability.py -v --tb=short
+
+          # Run autoscaling tests
+          python -m pytest .github/workflows/tests/test_autoscaling.py -v --tb=short -m "not slow"
+
+      - name: Debug observability stack on failure
+        if: failure()
+        run: |
+          ./scripts/debug-deployment.sh
+
+      - name: Cleanup observability test
+        if: always()
+        run: |
+          helm uninstall "$RELEASE_NAME" -n eoapi || true
+          kubectl delete namespace eoapi || true

.github/workflows/tests/conftest.py

@@ -1,40 +1,55 @@
-import pytest
 import os
-import psycopg2
-import psycopg2.extensions
 
+import pytest
+
+# Import psycopg2 conditionally to avoid import errors when not needed
+try:
+    import psycopg2
+    import psycopg2.extensions
 
-@pytest.fixture(scope='session')
+    PSYCOPG2_AVAILABLE = True
+except ImportError:
+    PSYCOPG2_AVAILABLE = False
+
+
+@pytest.fixture(scope="session")
 def raster_endpoint():
-    return os.getenv('RASTER_ENDPOINT', "http://127.0.0.1/raster")
+    return os.getenv("RASTER_ENDPOINT", "http://127.0.0.1/raster")
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
 def vector_endpoint():
-    return os.getenv('VECTOR_ENDPOINT', "http://127.0.0.1/vector")
+    return os.getenv("VECTOR_ENDPOINT", "http://127.0.0.1/vector")
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
 def stac_endpoint():
-    return os.getenv('STAC_ENDPOINT', "http://127.0.0.1/stac")
+    return os.getenv("STAC_ENDPOINT", "http://127.0.0.1/stac")
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture(scope="session")
 def db_connection():
     """Create database connection for testing."""
+    if not PSYCOPG2_AVAILABLE:
+        pytest.skip(
+            "psycopg2 not available - install with: pip install psycopg2-binary"
+        )
+
     # Require all database connection parameters to be explicitly set
-    required_vars = ['PGHOST', 'PGPORT', 'PGDATABASE', 'PGUSER', 'PGPASSWORD']
+    required_vars = ["PGHOST", "PGPORT", "PGDATABASE", "PGUSER", "PGPASSWORD"]
     missing_vars = [var for var in required_vars if not os.getenv(var)]
 
     if missing_vars:
-        pytest.fail(f"Required environment variables not set: {', '.join(missing_vars)}")
+        pytest.fail(
+            f"Required environment variables not set: {', '.join(missing_vars)}"
+        )
 
     connection_params = {
-        'host': os.getenv('PGHOST'),
-        'port': int(os.getenv('PGPORT')),
-        'database': os.getenv('PGDATABASE'),
-        'user': os.getenv('PGUSER'),
-        'password': os.getenv('PGPASSWORD')
+        "host": os.getenv("PGHOST"),
+        "port": int(os.getenv("PGPORT")),
+        "database": os.getenv("PGDATABASE"),
+        "user": os.getenv("PGUSER"),
+        "password": os.getenv("PGPASSWORD"),
     }
 
     try: