Cache tiler stac item queries (microsoft#251)

* Cache tiler stac item queries

* Changes after testing

* regenerate requirements

* stac as pyproject.toml

* clean up references to setup.py

* format

* Use same cache key between STAC and tiler

* Do not write from tiler

* fix the lines

Author: ghidalgo3

.flake8

@@ -4,5 +4,4 @@ extend-ignore = E203, W503
 exclude =
     .git
     __pycache__
-    setup.py
     .venv

README.md

@@ -84,7 +84,7 @@ After migrations and development database loading are in place, you can rebuild
 ./scripts/update
 ```
 
-`pip` dependencies in `setup.py` are collected and installed through requirements files.
+`pip` dependencies in `pyproject.toml` are collected and installed through requirements files.
 If you modify dependencies, run `./scripts/generate-requirements` to regenerate
 `requirements-*.txt` used by Dockerfiles otherwise your dependency change will not
 be realized.

deployment/Dockerfile

@@ -1,51 +1,44 @@
-FROM ubuntu:20.04
-
-RUN apt-get update --fix-missing
-RUN DEBIAN_FRONTEND=noninteractive apt-get install -y wget unzip curl gnupg \
-    apt-transport-https \
-    python3-pip \
+FROM mcr.microsoft.com/azurelinux/base/python:3.12
+
+RUN tdnf install -y \
+    ca-certificates \
+    build-essential \
+    tar \
+    wget \
+    unzip \
     jq \
     git \
-    libicu66
+    azure-cli \
+    && tdnf clean all
 
 # Install Azure Function Tools
-
-RUN curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/microsoft.gpg
-RUN echo "deb [arch=amd64] https://packages.microsoft.com/repos/microsoft-ubuntu-focal-prod focal main" \
-    > /etc/apt/sources.list.d/dotnetdev.list
-
-RUN apt-get update && apt-get install -y azure-functions-core-tools-4
-
-# Install Terraform 1.8.2
-
-RUN wget -O terraform.zip https://releases.hashicorp.com/terraform/1.8.2/terraform_1.8.2_linux_amd64.zip
-RUN unzip terraform.zip
-RUN mv terraform /usr/local/bin
+RUN wget https://github.com/Azure/azure-functions-core-tools/releases/download/4.0.5700/Azure.Functions.Cli.linux-x64.4.0.5700.zip \
+    && unzip Azure.Functions.Cli.linux-x64.4.0.5700.zip -d /usr/local/azure-functions-core-tools-4 \
+    && chmod +x /usr/local/azure-functions-core-tools-4/func \
+    && chmod +x /usr/local/azure-functions-core-tools-4/gozip \
+    && ln -s /usr/local/azure-functions-core-tools-4/func /usr/local/bin/func \
+    && ln -s /usr/local/azure-functions-core-tools-4/gozip /usr/local/bin/gozip
+
+# Install Terraform
+RUN wget -O terraform.zip https://releases.hashicorp.com/terraform/1.11.2/terraform_1.11.2_linux_amd64.zip \
+    && unzip terraform.zip \
+    && mv terraform /usr/local/bin
 
 # Install kubectl
-
-RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
-RUN install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
-
+RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
+    && install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
 
 # Install Helm
-
-RUN curl https://baltocdn.com/helm/signing.asc |  apt-key add -
-RUN echo "deb https://baltocdn.com/helm/stable/debian/ all main" | tee /etc/apt/sources.list.d/helm-stable-debian.list
-RUN apt-get update
-RUN apt-get install helm=3.14.0-1
+RUN wget https://get.helm.sh/helm-v3.14.4-linux-amd64.tar.gz \
+    && tar -zxvf helm-v3.14.4-linux-amd64.tar.gz \
+    && mv linux-amd64/helm /usr/local/bin/helm
 
 # Install kubelogin
-
-RUN curl -sL https://github.com/Azure/kubelogin/releases/download/v0.0.18/kubelogin-linux-amd64.zip --output kubelogin.zip \
+RUN curl -sL https://github.com/Azure/kubelogin/releases/download/v0.2.8/kubelogin-linux-amd64.zip --output kubelogin.zip \
     && unzip -j kubelogin.zip bin/linux_amd64/kubelogin -d /usr/local/bin/ \
     && rm -rf kubelogin.zip
 
-# Install azure client
-RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash
-
 # Install Jinja
-RUN pip3 install Jinja2 pyyaml==6.0
-
+RUN pip install Jinja2 pyyaml==6.0.2
 
 WORKDIR /opt/src

deployment/bin/deploy

@@ -176,6 +176,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
         --wait \
         --timeout 2m0s \
         -f ${DEPLOY_VALUES_FILE} \
+        --debug 
 
     echo "================"
     echo "==== Tiler ====="
@@ -188,6 +189,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
         --wait \
         --timeout 2m0s \
         -f ${DEPLOY_VALUES_FILE} \
+        --debug 
 
     echo "=================="
     echo "==== Ingress ====="
@@ -199,7 +201,8 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
         --kube-context "${KUBE_CONTEXT}" \
         --wait \
         --timeout 2m0s \
-        -f ${DEPLOY_VALUES_FILE}
+        -f ${DEPLOY_VALUES_FILE} \
+        --debug 
 
     echo "Installing ingress-nginx..."
     helm upgrade --install nginx-ingress helm/ingress-nginx-4.8.3.tgz \
@@ -215,7 +218,8 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then
         --version "4.8.3"\
         --wait \
         --timeout 2m0s \
-        -f bin/nginx-values.yaml
+        -f bin/nginx-values.yaml \
+        --debug 
 
     #########################
     # Deploy Azure Function #

deployment/terraform/resources/ai.tf

@@ -3,4 +3,10 @@ resource "azurerm_application_insights" "pc_application_insights" {
   location            = azurerm_resource_group.pc.location
   resource_group_name = azurerm_resource_group.pc.name
   application_type    = "web"
+
+  lifecycle {
+    ignore_changes = [
+      workspace_id
+    ]
+  }
 }

docker-compose.yml

@@ -117,7 +117,7 @@ services:
 
   redis:
     image: redis:6.2.6-buster
-    command: redis-server --port 6380 --requirepass devcache
+    command: redis-server --port 6380 --requirepass devcache --loglevel debug
     ports:
       - "6380:6380"
     volumes:

pccommon/pccommon/constants.py

@@ -2,6 +2,8 @@
 
 from opencensus.trace.attributes_helper import COMMON_ATTRIBUTES
 
+CACHE_KEY_ITEM = "/item"
+
 DEFAULT_COLLECTION_CONFIG_TABLE_NAME = "collectionconfig"
 DEFAULT_CONTAINER_CONFIG_TABLE_NAME = "containerconfig"
 DEFAULT_IP_EXCEPTION_CONFIG_TABLE_NAME = "ipexceptionlist"

pccommon/pccommon/redis.py

@@ -14,6 +14,7 @@
 from pccommon.config.core import PCAPIsConfig
 from pccommon.constants import (
     BACKPRESSURE_KEY_PREFIX,
+    CACHE_KEY_ITEM,
     HTTP_429_TOO_MANY_REQUESTS,
     RATE_LIMIT_KEY_PREFIX,
 )
@@ -104,9 +105,15 @@ async def register_scripts(state: State) -> None:
 
 
 async def cached_result(
-    fn: Callable[[], Coroutine[Any, Any, T]], cache_key: str, request: Request
+    fn: Callable[[], Coroutine[Any, Any, T]],
+    cache_key: str,
+    request: Request,
+    read_only: bool = False,
 ) -> T:
-    """Either get the result from redis or run the function and cache the result."""
+    """Either get the result from redis or run the function and cache the result.
+
+    If `read_only` is True, only attempt to read from the cache, do not write to it.
+    """
     host = request.url.hostname
     host_cache_key = f"{cache_key}:{host}"
     settings = PCAPIsConfig.from_environment()
@@ -124,7 +131,7 @@ async def cached_result(
     except Exception as e:
         # Don't fail on redis failure
         logger.error(
-            f"Error in cache: {e}",
+            f"Error in cache read: {e}",
             extra=get_custom_dimensions({"cache_key": host_cache_key}, request),
         )
         if settings.debug:
@@ -139,14 +146,19 @@ async def cached_result(
             {"cache_key": host_cache_key, "duration": f"{te - ts:0.4f}"}, request
         ),
     )
+    if read_only:
+        return result
+
     try:
         if r:
             await r.set(host_cache_key, orjson.dumps(result), settings.redis_ttl)
     except Exception as e:
         # Don't fail on redis failure
         logger.error(
-            f"Error in cache: {e}",
-            extra=get_custom_dimensions({"cache_key": host_cache_key}, request),
+            f"Error in cache write: {e}",
+            extra=get_custom_dimensions(
+                {"cache_key": host_cache_key, "cache_value_type": type(result)}, request
+            ),
         )
         if settings.debug:
             raise
@@ -321,3 +333,8 @@ async def _wrapper(*args: Any, **kwargs: Any) -> T:
         return _wrapper
 
     return _decorator
+
+
+def stac_item_cache_key(collection: str, item: str) -> str:
+    """Generate a cache key for a STAC item."""
+    return f"{CACHE_KEY_ITEM}:{collection}:{item}"

pcstac/pcstac/client.py

@@ -18,15 +18,14 @@
 
 from pccommon.config import get_all_render_configs, get_render_config
 from pccommon.config.collections import DefaultRenderConfig
-from pccommon.constants import DEFAULT_COLLECTION_REGION
+from pccommon.constants import CACHE_KEY_ITEM, DEFAULT_COLLECTION_REGION
 from pccommon.logging import get_custom_dimensions
-from pccommon.redis import back_pressure, cached_result, rate_limit
+from pccommon.redis import back_pressure, cached_result, rate_limit, stac_item_cache_key
 from pccommon.tracing import add_stac_attributes_from_search
 from pcstac.config import API_DESCRIPTION, API_LANDING_PAGE_ID, API_TITLE, get_settings
 from pcstac.contants import (
     CACHE_KEY_COLLECTION,
     CACHE_KEY_COLLECTIONS,
-    CACHE_KEY_ITEM,
     CACHE_KEY_ITEMS,
     CACHE_KEY_LANDING_PAGE,
     CACHE_KEY_SEARCH,
@@ -288,7 +287,7 @@ async def _fetch() -> Item:
             )
             return item
 
-        cache_key = f"{CACHE_KEY_ITEM}:{collection_id}:{item_id}"
+        cache_key = stac_item_cache_key(collection_id, item_id)
         return await cached_result(_fetch, cache_key, request)
 
     @classmethod

pcstac/pcstac/contants.py

@@ -1,7 +1,6 @@
 CACHE_KEY_COLLECTIONS = "/collections"
 CACHE_KEY_COLLECTION = "/collection"
 CACHE_KEY_ITEMS = "/items"
-CACHE_KEY_ITEM = "/item"
 CACHE_KEY_QUERYABLES = "/queryables"
 CACHE_KEY_SEARCH = "/search"
 CACHE_KEY_LANDING_PAGE = "/landing-page"