diff --git a/.flake8 b/.flake8 index 419d1104..e034c121 100644 --- a/.flake8 +++ b/.flake8 @@ -4,4 +4,5 @@ extend-ignore = E203, W503 exclude = .git __pycache__ - setup.py \ No newline at end of file + setup.py + .venv \ No newline at end of file diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index c33e87be..ea8068ec 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -32,7 +32,7 @@ jobs: - name: Get image tag id: get_image_tag - run: + run: case "${GITHUB_REF}" in *tags*) echo "tag=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT ; @@ -57,7 +57,7 @@ jobs: - build_and_publish steps: - uses: actions/checkout@v3 - + - name: Log in with Azure uses: azure/login@v1 with: @@ -86,4 +86,4 @@ jobs: ARM_CLIENT_ID: ${{ fromJSON(secrets.SECURE_AZURE_CREDENTIALS).clientId }} ARM_SUBSCRIPTION_ID: ${{ fromJSON(secrets.SECURE_AZURE_CREDENTIALS).subscriptionId }} ARM_TENANT_ID: ${{ fromJSON(secrets.SECURE_AZURE_CREDENTIALS).tenantId }} - ARM_USE_OIDC: true \ No newline at end of file + ARM_USE_OIDC: true diff --git a/deployment/README.md b/deployment/README.md index 1550e39e..cd33bb2b 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -10,6 +10,8 @@ The logic for the deployment workflow is encapsulated in the [bin/deploy](bin/de scripts/console --deploy ``` +To have access to the remote backend terraform state, the identity (App Registration in CI, or local corp credential if local) will need to have the `Storage Blob Data Owner` role on the `pctesttfstate` storage account. + ## Manual resources ### Deployment secrets Key Vault @@ -40,25 +42,27 @@ Container Registry repo where you published your local images: - `ACR_TILER_REPO` - `IMAGE_TAG` -__Note:__ Remember to bring down your resources after testing with `terraform destroy`! +**Note:** Remember to bring down your resources after testing with `terraform destroy`! ## Loading configuration data Configuration data is stored in Azure Storage Tables. Use the `pcapis` command line interface that is installed with the `pccommon` package to load data. For example: +```console +> az login # Use an account that has "Storage Table Data Contributor" on the account +> pcapis load -t collection --account pctapissatyasa --table collectionconfig --file pccommon/tests/data-files/collection_config.json ``` -> pcapis load -t collection --sas "${SAS_TOKEN}" --account pctapissatyasa --table collectionconfig --file pccommon/tests/data-files/collection_config.json -``` + To dump a single collection config, use: -``` -> pcapis dump -t collection --sas "${SAS_TOKEN}" --account pctapissatyasa --table collectionconfig --id naip +```console +> pcapis dump -t collection --account pctapissatyasa --table collectionconfig --id naip ``` For container configs, you must also specify the container account name used as the Partition Key: -``` -> pcapis dump -t collection --sas "${SAS_TOKEN}" --account pctapissatyasa --table containerconfig --id naip --container-account naipeuwest +```console +> pcapis dump -t collection --account pctapissatyasa --table containerconfig --id naip --container-account naipeuwest ``` Using the `load` command on a single dump file for either config will update the single row. diff --git a/deployment/bin/deploy b/deployment/bin/deploy index af776d1f..5e12133e 100755 --- a/deployment/bin/deploy +++ b/deployment/bin/deploy @@ -14,6 +14,7 @@ function usage() { Deploys the project infrastructure. -t TERRAFORM_DIR: The terraform directory. Required. +-y: Auto approve the terraform changes. --plan: Only run Terraform plan. --skip-tf: Skips Terraform apply. Will still gather terraform output " @@ -37,6 +38,10 @@ while [[ "$#" -gt 0 ]]; do case $1 in PLAN_ONLY=1 shift ;; + -y) + AUTO_APPROVE=-auto-approve + shift + ;; --help) usage exit 0 @@ -49,10 +54,29 @@ while [[ "$#" -gt 0 ]]; do case $1 in ;; esac done +# Always disable shared access keys on script exit +trap disable_shared_access_keys EXIT + ################################### # Check and configure environment # ################################### +# Enable shared access keys on storage accounts that must have properties read +# [storage_account]=resource_group +declare -A SAK_STORAGE_ACCOUNTS +SAK_STORAGE_ACCOUNTS=( + ["pctapisstagingsa"]="pct-apis-westeurope-staging_rg" + ["pcfilestest"]="pc-test-manual-resources" +) + +# Add client IP to firewall for storage accounts that must have properties read +# [storage_account]=resource_group +declare -A FW_STORAGE_ACCOUNTS +FW_STORAGE_ACCOUNTS=( + ["pctesttfstate"]="pc-test-manual-resources" + ["pctapisstagingsa"]="pct-apis-westeurope-staging_rg" +) + if [[ -z ${TERRAFORM_DIR} ]]; then echo "Must pass in TERRAFORM_DIR with -t" exit 1 @@ -73,15 +97,21 @@ setup_env echo "===== Running Deploy =====" echo "IMAGE_TAG: ${IMAGE_TAG}" +if [ -z "$ARM_CLIENT_ID" ]; then + export ARM_CLIENT_ID=$(az account show --query user.name -o tsv) + echo "Using Azure CLI auth with username: ${ARM_CLIENT_ID}" +fi + + # --------------------------------------------------- if [ "${BASH_SOURCE[0]}" = "${0}" ]; then ######################### - # Add IP to KV firewall # + # Add IP to firewalls # ######################### - bin/kv_add_ip + add_ip_to_firewalls ##################### # Deploy Terraform # @@ -91,6 +121,9 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then if [[ "${SKIP_TF}" != 1 ]]; then echo "Deploying infrastructure with Terraform..." + + enable_shared_access_keys + terraform init --upgrade if [ "${PLAN_ONLY}" ]; then @@ -98,7 +131,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then exit 0 fi - terraform apply -auto-approve + terraform apply "$AUTO_APPROVE" fi # Gather terraform output @@ -107,10 +140,10 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then popd ############################## - # Remove IP from KV firewall # + # Remove IP from firewalls # ############################## - bin/kv_rmv_ip + remove_ip_from_firewalls ############################ # Render Helm chart values # @@ -142,7 +175,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then --kube-context "${KUBE_CONTEXT}" \ --wait \ --timeout 2m0s \ - -f ${DEPLOY_VALUES_FILE} + -f ${DEPLOY_VALUES_FILE} \ echo "================" echo "==== Tiler =====" @@ -154,7 +187,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then --kube-context "${KUBE_CONTEXT}" \ --wait \ --timeout 2m0s \ - -f ${DEPLOY_VALUES_FILE} + -f ${DEPLOY_VALUES_FILE} \ echo "==================" echo "==== Ingress =====" diff --git a/deployment/bin/kv_add_ip b/deployment/bin/kv_add_ip deleted file mode 100755 index b507b52b..00000000 --- a/deployment/bin/kv_add_ip +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -set -e - -source bin/lib - -if [[ "${CI}" ]]; then - set -x -fi - -function usage() { - echo -n \ - "Usage: $(basename "$0") -Add runner public IP to Key Vault firewall allow list -" -} - -while [[ "$#" -gt 0 ]]; do case $1 in - *) - usage "Unknown parameter passed: $1" - shift - shift - ;; - esac done - - -if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - - cidr=$(get_cidr_range) - - az keyvault network-rule add \ - -g ${KEY_VAULT_RESOURCE_GROUP_NAME} \ - -n ${KEY_VAULT_NAME} \ - --ip-address $cidr \ - --subscription ${ARM_SUBSCRIPTION_ID} - -fi diff --git a/deployment/bin/kv_rmv_ip b/deployment/bin/kv_rmv_ip deleted file mode 100755 index dddc2401..00000000 --- a/deployment/bin/kv_rmv_ip +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -set -e - -source bin/lib - -if [[ "${CI}" ]]; then - set -x -fi - -function usage() { - echo -n \ - "Usage: $(basename "$0") -Remove runner public IP from Key Vault firewall allow list -" -} - -while [[ "$#" -gt 0 ]]; do case $1 in - *) - usage "Unknown parameter passed: $1" - shift - shift - ;; - esac done - - -if [ "${BASH_SOURCE[0]}" = "${0}" ]; then - - cidr=$(get_cidr_range) - - az keyvault network-rule remove \ - -g ${KEY_VAULT_RESOURCE_GROUP_NAME} \ - -n ${KEY_VAULT_NAME} \ - --ip-address $cidr \ - --subscription ${ARM_SUBSCRIPTION_ID} - -fi diff --git a/deployment/bin/lib b/deployment/bin/lib index 329e682b..c2c7cbf5 100755 --- a/deployment/bin/lib +++ b/deployment/bin/lib @@ -131,3 +131,96 @@ function get_cidr_range() { IFS='.' read -r -a ip_parts <<< "$runnerIpAddress" echo "${ip_parts[0]}.${ip_parts[1]}.0.0/16" } + +function disable_shared_access_keys() { + echo "Disabling shared access key on storage account..." + + for SAK_STORAGE_ACCOUNT in "${!SAK_STORAGE_ACCOUNTS[@]}"; do + SAK_RESOURCE_GROUP=${SAK_STORAGE_ACCOUNTS[$SAK_STORAGE_ACCOUNT]} + + az storage account update \ + --name ${SAK_STORAGE_ACCOUNT} \ + --resource-group ${SAK_RESOURCE_GROUP} \ + --allow-shared-key-access false \ + --subscription ${ARM_SUBSCRIPTION_ID} \ + --output none + + if [ $? -ne 0 ]; then + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + echo "WARNING: Failed to turn off shared key access on the storage account." + echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" + exit 2 + fi + done +} + +function enable_shared_access_keys() { + # Terraform isn't able to read all resources from a storage account if shared key access is disabled + # so while we're deploying, we need to enable it. Since we haven't run TF yet, we don't have the name of the account + # so they are hardcoded here. This is a temporary workaround until this is resolved + # https://github.com/hashicorp/terraform-provider-azurerm/issues/25218 + + echo "Enabling shared key access for storage accounts..." + for SAK_STORAGE_ACCOUNT in "${!SAK_STORAGE_ACCOUNTS[@]}"; do + SAK_RESOURCE_GROUP=${SAK_STORAGE_ACCOUNTS[$SAK_STORAGE_ACCOUNT]} + + echo " - ${SAK_RESOURCE_GROUP}.${SAK_STORAGE_ACCOUNT}" + az storage account update \ + --name ${SAK_STORAGE_ACCOUNT} \ + --resource-group ${SAK_RESOURCE_GROUP} \ + --allow-shared-key-access true \ + --subscription ${ARM_SUBSCRIPTION_ID} \ + --output none + done + + sleep 10 +} + +function add_ip_to_firewalls() { + cidr=$(get_cidr_range) + + echo "Adding IP $cidr to Key Vault firewall allow list..." + az keyvault network-rule add \ + -g "${KEY_VAULT_RESOURCE_GROUP_NAME}" \ + -n "${KEY_VAULT_NAME}" \ + --ip-address "$cidr" \ + --subscription "${ARM_SUBSCRIPTION_ID}" \ + --output none + + # Also add the IP to the terraform state storage account + for FW_STORAGE_ACCOUNT in "${!FW_STORAGE_ACCOUNTS[@]}"; do + FW_RESOURCE_GROUP=${FW_STORAGE_ACCOUNTS[$FW_STORAGE_ACCOUNT]} + echo "Adding IP $cidr to ${FW_STORAGE_ACCOUNT} Storage firewall allow list..." + az storage account network-rule add \ + -g "${FW_RESOURCE_GROUP}" \ + -n "${FW_STORAGE_ACCOUNT}" \ + --ip-address "$cidr" \ + --subscription "${ARM_SUBSCRIPTION_ID}" \ + --output none + done + + sleep 10 +} + +function remove_ip_from_firewalls() { + cidr=$(get_cidr_range) + + echo "Removing IP $cidr from Key Vault firewall allow list..." + az keyvault network-rule remove \ + -g ${KEY_VAULT_RESOURCE_GROUP_NAME} \ + -n ${KEY_VAULT_NAME} \ + --ip-address $cidr \ + --subscription ${ARM_SUBSCRIPTION_ID} \ + --output none + + for FW_STORAGE_ACCOUNT in "${!FW_STORAGE_ACCOUNTS[@]}"; do + FW_RESOURCE_GROUP=${FW_STORAGE_ACCOUNTS[$FW_STORAGE_ACCOUNT]} + echo "Removing IP $cidr from ${FW_STORAGE_ACCOUNT} Storage firewall allow list..." + az storage account network-rule remove \ + -g ${FW_RESOURCE_GROUP} \ + -n ${FW_STORAGE_ACCOUNT} \ + --ip-address $cidr \ + --subscription ${ARM_SUBSCRIPTION_ID} \ + --output none + done +} diff --git a/deployment/docker-compose.yml b/deployment/docker-compose.yml index 1e20e1a0..da67cbbe 100644 --- a/deployment/docker-compose.yml +++ b/deployment/docker-compose.yml @@ -8,11 +8,11 @@ services: environment: - ACR_STAC_REPO=${ACR_STAC_REPO:-pccomponentstest.azurecr.io/planetary-computer-apis/stac} - ACR_TILER_REPO=${ACR_TILER_REPO:-pccomponentstest.azurecr.io/planetary-computer-apis/tiler} - - IMAGE_TAG + - IMAGE_TAG=${IMAGE_TAG:-latest} - GIT_COMMIT - ARM_SUBSCRIPTION_ID=${ARM_SUBSCRIPTION_ID:-a84a690d-585b-4c7c-80d9-851a48af5a50} - - ARM_TENANT_ID + - ARM_TENANT_ID=${ARM_TENANT_ID:-72f988bf-86f1-41af-91ab-2d7cd011db47} - ARM_CLIENT_ID - ARM_USE_OIDC - ARM_OIDC_TOKEN diff --git a/deployment/helm/deploy-values.template.yaml b/deployment/helm/deploy-values.template.yaml index f8b33b26..443cd857 100644 --- a/deployment/helm/deploy-values.template.yaml +++ b/deployment/helm/deploy-values.template.yaml @@ -42,6 +42,11 @@ stac: replicaCount: "{{ tf.stac_replica_count }}" podAnnotations: "pc/gitsha": "{{ env.GIT_COMMIT }}" + useWorkloadIdentity: true + serviceAccount: + annotations: + "azure.workload.identity/client-id": {{ tf.cluster_stac_identity_client_id }} + "azure.workload.identity/tenant-id": {{ tf.tenant_id }} appRootPath: "/stac" port: "80" @@ -86,7 +91,6 @@ tiler: storage: account_name: "{{ tf.storage_account_name }}" - account_key: "{{ tf.storage_account_key }}" collection_config_table_name: "{{ tf.collection_config_table_name }}" container_config_table_name: "{{ tf.container_config_table_name }}" ip_exception_config_table_name: "{{ tf.ip_exception_config_table_name }}" diff --git a/deployment/helm/published/planetary-computer-stac/templates/_helpers.tpl b/deployment/helm/published/planetary-computer-stac/templates/_helpers.tpl index 47b5fa14..433a2779 100644 --- a/deployment/helm/published/planetary-computer-stac/templates/_helpers.tpl +++ b/deployment/helm/published/planetary-computer-stac/templates/_helpers.tpl @@ -42,6 +42,7 @@ app.kubernetes.io/instance: {{ .Release.Name }} Common labels */}} {{- define "pcstac.labels" -}} +azure.workload.identity/use: {{ .Values.stac.deploy.useWorkloadIdentity | quote}} helm.sh/chart: {{ include "pcstac.chart" . }} {{ include "pcstac.selectorLabels" . }} {{- if .Chart.AppVersion }} diff --git a/deployment/helm/published/planetary-computer-stac/templates/deployment.yaml b/deployment/helm/published/planetary-computer-stac/templates/deployment.yaml index 625f9eb0..dacda7ca 100644 --- a/deployment/helm/published/planetary-computer-stac/templates/deployment.yaml +++ b/deployment/helm/published/planetary-computer-stac/templates/deployment.yaml @@ -19,7 +19,7 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} labels: - {{- include "pcstac.selectorLabels" . | nindent 8 }} + {{- include "pcstac.labels" . | nindent 8 }} spec: {{- with .Values.stac.deploy.imagePullSecrets }} imagePullSecrets: @@ -89,20 +89,14 @@ spec: value: "{{ .Values.stac.debug }}" - name: "PCAPIS_COLLECTION_CONFIG__ACCOUNT_NAME" value: "{{ .Values.storage.account_name }}" - - name: "PCAPIS_COLLECTION_CONFIG__ACCOUNT_KEY" - value: "{{ .Values.storage.account_key }}" - name: "PCAPIS_COLLECTION_CONFIG__TABLE_NAME" value: "{{ .Values.storage.collection_config_table_name }}" - name: "PCAPIS_CONTAINER_CONFIG__ACCOUNT_NAME" value: "{{ .Values.storage.account_name }}" - - name: "PCAPIS_CONTAINER_CONFIG__ACCOUNT_KEY" - value: "{{ .Values.storage.account_key }}" - name: "PCAPIS_CONTAINER_CONFIG__TABLE_NAME" value: "{{ .Values.storage.container_config_table_name }}" - name: "PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_NAME" value: "{{ .Values.storage.account_name }}" - - name: "PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_KEY" - value: "{{ .Values.storage.account_key }}" - name: "PCAPIS_IP_EXCEPTION_CONFIG__TABLE_NAME" value: "{{ .Values.storage.ip_exception_config_table_name }}" - name: "PCAPIS_REDIS_HOSTNAME" diff --git a/deployment/helm/published/planetary-computer-stac/templates/serviceaccount.yaml b/deployment/helm/published/planetary-computer-stac/templates/serviceaccount.yaml index 3177d7fa..511dc933 100644 --- a/deployment/helm/published/planetary-computer-stac/templates/serviceaccount.yaml +++ b/deployment/helm/published/planetary-computer-stac/templates/serviceaccount.yaml @@ -6,7 +6,7 @@ metadata: name: {{ include "pcstac.serviceAccountName" . }} labels: {{- include "pcstac.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} + {{- with .Values.stac.deploy.serviceAccount.annotations }} annotations: {{- toYaml . | nindent 4 }} {{- end }} diff --git a/deployment/helm/published/planetary-computer-stac/values.yaml b/deployment/helm/published/planetary-computer-stac/values.yaml index 46dd7565..e4f67b10 100644 --- a/deployment/helm/published/planetary-computer-stac/values.yaml +++ b/deployment/helm/published/planetary-computer-stac/values.yaml @@ -56,6 +56,10 @@ stac: affinity: {} autoscaling: enabled: false + useWorkloadIdentity: false + serviceAccount: + annotations: {} + cert: privateKeySecretRef: "letsencrypt-staging" diff --git a/deployment/helm/published/planetary-computer-tiler/templates/deployment.yaml b/deployment/helm/published/planetary-computer-tiler/templates/deployment.yaml index 9f3206dd..f7573f26 100644 --- a/deployment/helm/published/planetary-computer-tiler/templates/deployment.yaml +++ b/deployment/helm/published/planetary-computer-tiler/templates/deployment.yaml @@ -85,20 +85,14 @@ spec: value: "{{ .Values.tiler.default_max_items_per_tile}}" - name: "PCAPIS_COLLECTION_CONFIG__ACCOUNT_NAME" value: "{{ .Values.storage.account_name }}" - - name: "PCAPIS_COLLECTION_CONFIG__ACCOUNT_KEY" - value: "{{ .Values.storage.account_key }}" - name: "PCAPIS_COLLECTION_CONFIG__TABLE_NAME" value: "{{ .Values.storage.collection_config_table_name }}" - name: "PCAPIS_CONTAINER_CONFIG__ACCOUNT_NAME" value: "{{ .Values.storage.account_name }}" - - name: "PCAPIS_CONTAINER_CONFIG__ACCOUNT_KEY" - value: "{{ .Values.storage.account_key }}" - name: "PCAPIS_CONTAINER_CONFIG__TABLE_NAME" value: "{{ .Values.storage.container_config_table_name }}" - name: "PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_NAME" value: "{{ .Values.storage.account_name }}" - - name: "PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_KEY" - value: "{{ .Values.storage.account_key }}" - name: "PCAPIS_IP_EXCEPTION_CONFIG__TABLE_NAME" value: "{{ .Values.storage.ip_exception_config_table_name }}" - name: "PCAPIS_TABLE_VALUE_TTL" diff --git a/deployment/terraform/resources/aks.tf b/deployment/terraform/resources/aks.tf index 109e69f2..a9ac71a9 100644 --- a/deployment/terraform/resources/aks.tf +++ b/deployment/terraform/resources/aks.tf @@ -26,6 +26,10 @@ resource "azurerm_kubernetes_cluster" "pc" { vm_size = "Standard_DS2_v2" node_count = var.aks_node_count vnet_subnet_id = azurerm_subnet.node_subnet.id + + upgrade_settings { + max_surge = "10%" + } } identity { @@ -74,9 +78,31 @@ resource "azurerm_kubernetes_cluster" "pc" { } } +resource "azurerm_user_assigned_identity" "stac" { + name = "id-${local.prefix}-stac" + location = var.region + resource_group_name = azurerm_resource_group.pc.name +} + +resource "azurerm_federated_identity_credential" "stac" { + name = "federated-id-${local.prefix}" + resource_group_name = azurerm_resource_group.pc.name + audience = ["api://AzureADTokenExchange"] + issuer = azurerm_kubernetes_cluster.pc.oidc_issuer_url + subject = "system:serviceaccount:pc:planetary-computer-stac" + parent_id = azurerm_user_assigned_identity.stac.id + timeouts {} +} + +resource "azurerm_role_assignment" "cluster-stac-identity-storage-access" { + scope = azurerm_storage_account.pc.id + role_definition_name = "Storage Table Data Reader" + principal_id = azurerm_user_assigned_identity.stac.principal_id +} + # Workload Identity for tiler access to the Azure Maps account resource "azurerm_user_assigned_identity" "tiler" { - name = "id-${local.prefix}" + name = "id-${local.prefix}-tiler" location = var.region resource_group_name = azurerm_resource_group.pc.name } @@ -98,6 +124,12 @@ resource "azurerm_role_assignment" "cluster-identity-maps-render-token" { } +resource "azurerm_role_assignment" "cluster-tiler-identity-storage-access" { + scope = azurerm_storage_account.pc.id + role_definition_name = "Storage Table Data Reader" + principal_id = azurerm_user_assigned_identity.tiler.principal_id +} + # add the role to the identity the kubernetes cluster was assigned resource "azurerm_role_assignment" "network" { scope = azurerm_resource_group.pc.id diff --git a/deployment/terraform/resources/functions.tf b/deployment/terraform/resources/functions.tf index cdcf6ebf..463ef33c 100644 --- a/deployment/terraform/resources/functions.tf +++ b/deployment/terraform/resources/functions.tf @@ -1,35 +1,42 @@ -resource "azurerm_app_service_plan" "pc" { - name = "plan-${local.prefix}" +resource "azurerm_service_plan" "pc" { + name = "app-plan-${local.prefix}" location = azurerm_resource_group.pc.location resource_group_name = azurerm_resource_group.pc.name - kind = "functionapp" - reserved = true + os_type = "Linux" + + sku_name = "EP1" - sku { - tier = "Dynamic" - size = "Y1" - } } -resource "azurerm_function_app" "pcfuncs" { - name = "func-${local.prefix}" - location = azurerm_resource_group.pc.location - resource_group_name = azurerm_resource_group.pc.name - app_service_plan_id = azurerm_app_service_plan.pc.id - storage_account_name = azurerm_storage_account.pc.name - storage_account_access_key = azurerm_storage_account.pc.primary_access_key - https_only = true +resource "azurerm_linux_function_app" "pcfuncs" { + name = "func-${local.prefix}" + location = azurerm_resource_group.pc.location + resource_group_name = azurerm_resource_group.pc.name + service_plan_id = azurerm_service_plan.pc.id + storage_account_name = azurerm_storage_account.pc.name + + virtual_network_subnet_id = azurerm_subnet.function_subnet.id + + ftp_publish_basic_authentication_enabled = false + webdeploy_publish_basic_authentication_enabled = false + + + storage_uses_managed_identity = true + https_only = true identity { type = "SystemAssigned" } app_settings = { - "ENABLE_ORYX_BUILD" = "true", - "SCM_DO_BUILD_DURING_DEPLOYMENT" = "true", - "FUNCTIONS_WORKER_RUNTIME" = "python", - "APP_INSIGHTS_IKEY" = azurerm_application_insights.pc_application_insights.instrumentation_key, - "APPINSIGHTS_INSTRUMENTATIONKEY" = azurerm_application_insights.pc_application_insights.instrumentation_key, + "FUNCTIONS_WORKER_RUNTIME" = "python", + "APP_INSIGHTS_IKEY" = azurerm_application_insights.pc_application_insights.instrumentation_key, + + # Remote build + "BUILD_FLAGS" = "UseExpressBuild", + "ENABLE_ORYX_BUILD" = "true" + "SCM_DO_BUILD_DURING_DEPLOYMENT" = "1", + "XDG_CACHE_HOME" = "/tmp/.cache" "AzureWebJobsDisableHomepage" = true, # Animation Function @@ -41,20 +48,25 @@ resource "azurerm_function_app" "pcfuncs" { "IMAGE_OUTPUT_STORAGE_URL" = var.image_output_storage_url, "IMAGE_API_ROOT_URL" = var.funcs_data_api_url, "IMAGE_TILE_REQUEST_CONCURRENCY" = tostring(var.funcs_tile_request_concurrency), + + # IPBan function + "STORAGE_ACCOUNT_URL" = var.func_storage_account_url, + "BANNED_IP_TABLE" = var.banned_ip_table, + "LOG_ANALYTICS_WORKSPACE_ID" = var.prod_log_analytics_workspace_id, } - os_type = "linux" - version = "~4" site_config { - linux_fx_version = "PYTHON|3.8" - use_32_bit_worker_process = false - ftps_state = "Disabled" + vnet_route_all_enabled = true + application_insights_key = azurerm_application_insights.pc_application_insights.instrumentation_key + ftps_state = "Disabled" cors { allowed_origins = ["*"] } + application_stack { + python_version = "3.9" + } } - lifecycle { ignore_changes = [ tags @@ -62,18 +74,46 @@ resource "azurerm_function_app" "pcfuncs" { } } -# Note: this must be in the same subscription as the rest of the deployed infrastructure -data "azurerm_storage_container" "output" { - name = var.output_container_name - storage_account_name = var.output_storage_account_name + + +resource "azurerm_role_assignment" "function-app-storage-account-access" { + scope = azurerm_storage_account.pc.id + role_definition_name = "Storage Blob Data Owner" + principal_id = azurerm_linux_function_app.pcfuncs.identity[0].principal_id } resource "azurerm_role_assignment" "function-app-animation-container-access" { - scope = data.azurerm_storage_container.output.resource_manager_id + scope = data.azurerm_storage_account.output-storage-account.id role_definition_name = "Storage Blob Data Contributor" - principal_id = azurerm_function_app.pcfuncs.identity[0].principal_id + principal_id = azurerm_linux_function_app.pcfuncs.identity[0].principal_id + + depends_on = [ + azurerm_linux_function_app.pcfuncs + ] +} + +resource "azurerm_role_assignment" "function-app-storage-table-data-contributor" { + scope = azurerm_storage_account.pc.id + role_definition_name = "Storage Table Data Contributor" + principal_id = azurerm_linux_function_app.pcfuncs.identity[0].principal_id + + depends_on = [ + azurerm_linux_function_app.pcfuncs + ] +} + +data "azurerm_log_analytics_workspace" "prod_log_analytics_workspace" { + provider = azurerm.planetary_computer_subscription + name = var.prod_log_analytics_workspace_name + resource_group_name = var.pc_resources_rg +} + +resource "azurerm_role_assignment" "function-app-log-analytics-access" { + scope = data.azurerm_log_analytics_workspace.prod_log_analytics_workspace.id + role_definition_name = "Log Analytics Reader" + principal_id = azurerm_linux_function_app.pcfuncs.identity[0].principal_id depends_on = [ - azurerm_function_app.pcfuncs + azurerm_linux_function_app.pcfuncs ] } diff --git a/deployment/terraform/resources/output.tf b/deployment/terraform/resources/output.tf index 733ae520..f6c0c5f2 100644 --- a/deployment/terraform/resources/output.tf +++ b/deployment/terraform/resources/output.tf @@ -55,6 +55,10 @@ output "cluster_tiler_identity_client_id" { value = azurerm_user_assigned_identity.tiler.client_id } +output "cluster_stac_identity_client_id" { + value = azurerm_user_assigned_identity.stac.client_id +} + ## Ingress output "ingress_ip" { @@ -104,10 +108,6 @@ output "storage_account_name" { value = azurerm_storage_account.pc.name } -output "storage_account_key" { - value = azurerm_storage_account.pc.primary_access_key -} - output "collection_config_table_name" { value = azurerm_storage_table.collectionconfig.name } @@ -137,5 +137,5 @@ output "redis_port" { # Functions output "function_app_name" { - value = azurerm_function_app.pcfuncs.name + value = azurerm_linux_function_app.pcfuncs.name } diff --git a/deployment/terraform/resources/providers.tf b/deployment/terraform/resources/providers.tf index 5671a49f..d71323b0 100644 --- a/deployment/terraform/resources/providers.tf +++ b/deployment/terraform/resources/providers.tf @@ -1,6 +1,17 @@ -provider azurerm { +provider "azurerm" { features {} use_oidc = true + + # This could be used instead of temporarily enabling shared key access once + # this issue is resolved. + # https://github.com/hashicorp/terraform-provider-azurerm/issues/23142 + # storage_use_azuread = true +} + +provider "azurerm" { + alias = "planetary_computer_subscription" + subscription_id = "9da7523a-cb61-4c3e-b1d4-afa5fc6d2da9" + features {} } terraform { @@ -9,9 +20,9 @@ terraform { required_providers { azurerm = { source = "hashicorp/azurerm" - version = "3.97.1" + version = "3.108.0" } } } -data "azurerm_client_config" "current" {} \ No newline at end of file +data "azurerm_client_config" "current" {} diff --git a/deployment/terraform/resources/storage_account.tf b/deployment/terraform/resources/storage_account.tf index e55aa6d1..1ad494b0 100644 --- a/deployment/terraform/resources/storage_account.tf +++ b/deployment/terraform/resources/storage_account.tf @@ -6,8 +6,26 @@ resource "azurerm_storage_account" "pc" { account_replication_type = "LRS" min_tls_version = "TLS1_2" allow_nested_items_to_be_public = false + + network_rules { + default_action = "Deny" + virtual_network_subnet_ids = [azurerm_subnet.node_subnet.id, azurerm_subnet.function_subnet.id, data.azurerm_subnet.sas_node_subnet.id] + } + + # Disabling shared access keys breaks terraform's ability to do subsequent + # resource fetching during terraform plan. As a result, this property is + # ignored and managed outside of this apply session, via the deploy script. + # https://github.com/hashicorp/terraform-provider-azurerm/issues/25218 + + # shared_access_key_enabled = false + lifecycle { + ignore_changes = [ + shared_access_key_enabled, + ] + } } + # Tables resource "azurerm_storage_table" "collectionconfig" { @@ -24,3 +42,22 @@ resource "azurerm_storage_table" "ipexceptionlist" { name = "ipexceptionlist" storage_account_name = azurerm_storage_account.pc.name } + +resource "azurerm_storage_table" "blobstoragebannedip" { + name = "blobstoragebannedip" + storage_account_name = azurerm_storage_account.pc.name +} + +# Output storage account for function app, "pcfilestest" +data "azurerm_storage_account" "output-storage-account" { + name = var.output_storage_account_name + resource_group_name = var.pc_test_resources_rg + +} + +resource "azurerm_storage_account_network_rules" "pcfunc-vnet-access" { + storage_account_id = data.azurerm_storage_account.output-storage-account.id + + default_action = "Deny" + virtual_network_subnet_ids = [azurerm_subnet.function_subnet.id] +} diff --git a/deployment/terraform/resources/variables.tf b/deployment/terraform/resources/variables.tf index c760a708..9134bc30 100644 --- a/deployment/terraform/resources/variables.tf +++ b/deployment/terraform/resources/variables.tf @@ -11,6 +11,11 @@ variable "pc_test_resources_rg" { default = "pc-test-manual-resources" } +variable "pc_resources_rg" { + type = string + default = "pc-manual-resources" +} + variable "pc_test_resources_kv" { type = string default = "pc-test-deploy-secrets" @@ -123,6 +128,33 @@ variable "image_output_storage_url" { type = string } +variable "prod_log_analytics_workspace_name" { + type = string +} + +variable "prod_log_analytics_workspace_id" { + type = string +} + +variable "banned_ip_table" { + type = string +} + +variable "func_storage_account_url" { + type = string +} + +variable "sas_node_subnet_name" { + type = string +} + +variable "sas_node_subnet_virtual_network_name" { + type = string +} + +variable "sas_node_subnet_resource_group_name" { + type = string +} # ----------------- # Local variables diff --git a/deployment/terraform/resources/vnet.tf b/deployment/terraform/resources/vnet.tf index 917152b7..660441db 100644 --- a/deployment/terraform/resources/vnet.tf +++ b/deployment/terraform/resources/vnet.tf @@ -26,6 +26,31 @@ resource "azurerm_subnet" "cache_subnet" { service_endpoints = [] } +data "azurerm_subnet" "sas_node_subnet" { + name = var.sas_node_subnet_name + virtual_network_name = var.sas_node_subnet_virtual_network_name + resource_group_name = var.sas_node_subnet_resource_group_name +} + +resource "azurerm_subnet" "function_subnet" { + name = "${local.prefix}-functions-subnet" + virtual_network_name = azurerm_virtual_network.pc.name + resource_group_name = azurerm_resource_group.pc.name + + service_endpoints = ["Microsoft.Storage.Global"] + delegation { + name = "delegation" + service_delegation { + actions = [ + "Microsoft.Network/virtualNetworks/subnets/action", + ] + name = "Microsoft.Web/serverFarms" + } + } + + address_prefixes = ["10.3.0.0/26"] +} + resource "azurerm_network_security_group" "pc" { name = "${local.prefix}-security-group" location = azurerm_resource_group.pc.location @@ -53,3 +78,8 @@ resource "azurerm_subnet_network_security_group_association" "pc-cache" { subnet_id = azurerm_subnet.cache_subnet.id network_security_group_id = azurerm_network_security_group.pc.id } + +resource "azurerm_subnet_network_security_group_association" "pc-functions" { + subnet_id = azurerm_subnet.function_subnet.id + network_security_group_id = azurerm_network_security_group.pc.id +} diff --git a/deployment/terraform/staging/main.tf b/deployment/terraform/staging/main.tf index 269a2401..bfbb6baf 100644 --- a/deployment/terraform/staging/main.tf +++ b/deployment/terraform/staging/main.tf @@ -22,6 +22,14 @@ module "resources" { animation_output_storage_url = "https://pcfilestest.blob.core.windows.net/output/animations" image_output_storage_url = "https://pcfilestest.blob.core.windows.net/output/images" + prod_log_analytics_workspace_name = "pc-api-loganalytics" + prod_log_analytics_workspace_id = "78d48390-b6bb-49a9-b7fd-a86f6522e9c4" + func_storage_account_url = "https://pctapisstagingsa.table.core.windows.net/" + banned_ip_table = "blobstoragebannedip" + + sas_node_subnet_name = "pct-sas-westeurope-staging-node-subnet" + sas_node_subnet_virtual_network_name = "pct-sas-westeurope-staging-network" + sas_node_subnet_resource_group_name = "pct-sas-westeurope-staging_rg" } terraform { @@ -31,6 +39,7 @@ terraform { container_name = "pc-test-api" key = "pqe-apis.tfstate" use_oidc = true + use_azuread_auth = true } } diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 1f66fdb9..6204104d 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,5 +1,6 @@ services: stac-dev: + platform: linux/amd64 image: pc-apis-stac-dev build: context: . @@ -63,6 +64,7 @@ services: depends_on: - stac tiler-dev: + platform: linux/amd64 image: pc-apis-tiler-dev # For Mac OS M1 user, you'll need to add `platform: linux/amd64`. # see https://github.com/developmentseed/titiler/discussions/387#discussioncomment-1643110 diff --git a/docker-compose.yml b/docker-compose.yml index 98f3b1c4..461f31fc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,7 @@ services: stac: + platform: linux/amd64 image: pc-apis-stac build: context: . @@ -22,7 +23,7 @@ services: image: pc-apis-tiler # For Mac OS M1 user, you'll need to add `platform: linux/amd64`. # see https://github.com/developmentseed/titiler/discussions/387#discussioncomment-1643110 - # platform: linux/amd64 + platform: linux/amd64 build: context: . dockerfile: pctiler/Dockerfile @@ -43,6 +44,7 @@ services: funcs: image: pc-apis-funcs + platform: linux/amd64 build: context: . dockerfile: pcfuncs/Dockerfile @@ -53,6 +55,7 @@ services: - ./pccommon:/home/site/pccommon - ./pcfuncs:/home/site/wwwroot - .:/opt/src + - ~/.azure:/home/.azure nginx: image: pc-apis-nginx diff --git a/pc-funcs.dev.env b/pc-funcs.dev.env index 87bf075c..63b5f6ee 100644 --- a/pc-funcs.dev.env +++ b/pc-funcs.dev.env @@ -10,3 +10,7 @@ IMAGE_OUTPUT_STORAGE_URL="http://azurite:10000/devstoreaccount1/output/images" IMAGE_OUTPUT_ACCOUNT_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" IMAGE_API_ROOT_URL="https://planetarycomputer-staging.microsoft.com/api/data/v1" IMAGE_TILE_REQUEST_CONCURRENCY=2 + +STORAGE_ACCOUNT_URL=https://pctapisstagingsa.table.core.windows.net/ +BANNED_IP_TABLE=blobstoragebannedip +LOG_ANALYTICS_WORKSPACE_ID=78d48390-b6bb-49a9-b7fd-a86f6522e9c4 \ No newline at end of file diff --git a/pc-stac.dev.env b/pc-stac.dev.env index 1933729d..9a7ea0d7 100644 --- a/pc-stac.dev.env +++ b/pc-stac.dev.env @@ -15,17 +15,14 @@ USE_API_HYDRATE=TRUE # Azure Storage PCAPIS_COLLECTION_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1 PCAPIS_COLLECTION_CONFIG__ACCOUNT_NAME=devstoreaccount1 -PCAPIS_COLLECTION_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== PCAPIS_COLLECTION_CONFIG__TABLE_NAME=collectionconfig PCAPIS_CONTAINER_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1 PCAPIS_CONTAINER_CONFIG__ACCOUNT_NAME=devstoreaccount1 -PCAPIS_CONTAINER_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== PCAPIS_CONTAINER_CONFIG__TABLE_NAME=containerconfig PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1 PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_NAME=devstoreaccount1 -PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== PCAPIS_IP_EXCEPTION_CONFIG__TABLE_NAME=ipexceptionlist # Disable config and stac caching in development by setting TTL to 1 second diff --git a/pc-tiler.dev.env b/pc-tiler.dev.env index fa5e9153..830ba6ee 100644 --- a/pc-tiler.dev.env +++ b/pc-tiler.dev.env @@ -25,17 +25,14 @@ VECTORTILE_SA_BASE_URL=https://pcvectortiles.blob.core.windows.net # Azure Storage PCAPIS_COLLECTION_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1 PCAPIS_COLLECTION_CONFIG__ACCOUNT_NAME=devstoreaccount1 -PCAPIS_COLLECTION_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== PCAPIS_COLLECTION_CONFIG__TABLE_NAME=collectionconfig PCAPIS_CONTAINER_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1 PCAPIS_CONTAINER_CONFIG__ACCOUNT_NAME=devstoreaccount1 -PCAPIS_CONTAINER_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== PCAPIS_CONTAINER_CONFIG__TABLE_NAME=containerconfig PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_URL=http://azurite:10002/devstoreaccount1 PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_NAME=devstoreaccount1 -PCAPIS_IP_EXCEPTION_CONFIG__ACCOUNT_KEY=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw== PCAPIS_IP_EXCEPTION_CONFIG__TABLE_NAME=ipexceptionlist # Disable config and stac caching in development by setting TTL to 1 second diff --git a/pccommon/pccommon/blob.py b/pccommon/pccommon/blob.py index c4390355..b5fe0207 100644 --- a/pccommon/pccommon/blob.py +++ b/pccommon/pccommon/blob.py @@ -1,27 +1,22 @@ from typing import Dict, Optional, Union -from azure.identity import DefaultAzureCredential +from azure.identity import ManagedIdentityCredential from azure.storage.blob import ContainerClient +from pccommon.constants import AZURITE_ACCOUNT_KEY + def get_container_client( container_url: str, - sas_token: Optional[str] = None, - account_key: Optional[str] = None, ) -> ContainerClient: - credential: Optional[Union[str, Dict[str, str], DefaultAzureCredential]] = None - if account_key: - # Handle Azurite - if "devstoreaccount1" in container_url: - credential = { - "account_name": "devstoreaccount1", - "account_key": account_key, - } - else: - credential = account_key - elif sas_token: - credential = sas_token + credential: Optional[Union[Dict[str, str], ManagedIdentityCredential]] = None + # Handle Azurite + if container_url.startswith("http://azurite:"): + credential = { + "account_name": "devstoreaccount1", + "account_key": AZURITE_ACCOUNT_KEY, + } else: - credential = DefaultAzureCredential() + credential = ManagedIdentityCredential() return ContainerClient.from_container_url(container_url, credential=credential) diff --git a/pccommon/pccommon/cli.py b/pccommon/pccommon/cli.py index 8ee834c4..06c1d376 100644 --- a/pccommon/pccommon/cli.py +++ b/pccommon/pccommon/cli.py @@ -15,20 +15,13 @@ from pccommon.version import __version__ -def get_account_url(account: str, account_url: Optional[str]) -> str: - return account_url or f"https://{account}.table.core.windows.net" - - -def load( - sas: str, account: str, table: str, type: str, file: str, **kwargs: Any -) -> int: - account_url = get_account_url(account, kwargs.get("account_url")) +def load(account: str, table: str, type: str, file: str, **kwargs: Any) -> int: with open(file) as f: rows = json.load(f) if type == "collection": - col_config_table = CollectionConfigTable.from_sas_token( - account_url=account_url, sas_token=sas, table_name=table + col_config_table = CollectionConfigTable.from_environment( + account_name=account, table_name=table ) for coll_id, config in rows.items(): print("Loading config for collection", coll_id) @@ -41,8 +34,8 @@ def load( print("========================================") elif type == "container": - cont_config_table = ContainerConfigTable.from_sas_token( - account_url=account_url, sas_token=sas, table_name=table + cont_config_table = ContainerConfigTable.from_environment( + account_name=account, table_name=table ) for path, config in rows.items(): storage_account, container = path.split("/") @@ -56,40 +49,39 @@ def load( return 0 -def dump(sas: str, account: str, table: str, type: str, **kwargs: Any) -> int: +def dump(account: str, table: str, type: str, **kwargs: Any) -> int: output = kwargs.get("output") - account_url = get_account_url(account, kwargs.get("account_url")) id = kwargs.get("id") result: Dict[str, Dict[str, Any]] = {} if type == "collection": - col_config_table = CollectionConfigTable.from_sas_token( - account_url=account_url, sas_token=sas, table_name=table + col_config_table = CollectionConfigTable.from_environment( + account_name=account, table_name=table ) if id: col_config = col_config_table.get_config(id) assert col_config - result[id] = col_config.dict() + result[id] = col_config.model_dump() else: for _, collection_id, col_config in col_config_table.get_all(): assert collection_id assert col_config - result[collection_id] = col_config.dict() + result[collection_id] = col_config.model_dump() elif type == "container": - con_config_table = ContainerConfigTable.from_sas_token( - account_url=account_url, sas_token=sas, table_name=table + con_config_table = ContainerConfigTable.from_environment( + account_name=account, table_name=table ) if id: con_account = kwargs.get("container_account") assert con_account con_config = con_config_table.get_config(con_account, id) assert con_config - result[f"{con_account}/{id}"] = con_config.dict() + result[f"{con_account}/{id}"] = con_config.model_dump() else: for storage_account, container, con_config in con_config_table.get_all(): assert con_config - result[f"{storage_account}/{container}"] = con_config.dict() + result[f"{storage_account}/{container}"] = con_config.model_dump() else: print(f"Unknown type: {type}") return 1 @@ -103,12 +95,11 @@ def dump(sas: str, account: str, table: str, type: str, **kwargs: Any) -> int: return 0 -def add_ip_exception(sas: str, account: str, table: str, **kwargs: Any) -> int: +def add_ip_exception(account: str, table: str, **kwargs: Any) -> int: ip_file = kwargs.get("file") ip = kwargs.get("ip") - account_url = get_account_url(account, kwargs.get("account_url")) - ip_table = IPExceptionListTable.from_sas_token( - account_url=account_url, sas_token=sas, table_name=table + ip_table = IPExceptionListTable.from_environment( + account_name=account, table_name=table ) if ip: print(f"Adding exception for IP {ip}...") @@ -139,11 +130,6 @@ def parse_args(args: List[str]) -> Optional[Dict[str, Any]]: subparsers = parser0.add_subparsers(dest="command") def add_common_opts(p: argparse.ArgumentParser, default_table: str) -> None: - p.add_argument( - "--sas", - help="SAS Token for the storage account.", - required=True, - ) p.add_argument("--account", help="Storage account name.", required=True) p.add_argument("--table", help="Table name.", default=default_table) p.add_argument( diff --git a/pccommon/pccommon/config/collections.py b/pccommon/pccommon/config/collections.py index c6ac12b6..bcbc145b 100644 --- a/pccommon/pccommon/config/collections.py +++ b/pccommon/pccommon/config/collections.py @@ -1,12 +1,11 @@ from enum import Enum from typing import Any, Dict, List, Optional, Tuple -import orjson from humps import camelize from pydantic import BaseModel, Field from pccommon.tables import ModelTableService -from pccommon.utils import get_param_str, orjson_dumps +from pccommon.utils import get_param_str class RenderOptionType(str, Enum): @@ -19,11 +18,13 @@ def __str__(self) -> str: class CamelModel(BaseModel): - class Config: - alias_generator = camelize - allow_population_by_field_name = True - json_loads = orjson.loads - json_dumps = orjson_dumps + + model_config = { + # TODO, see if we can use pydantic native function + # https://docs.pydantic.dev/latest/api/config/#pydantic.alias_generators.to_camel + "alias_generator": camelize, + "populate_by_name": True, + } class VectorTileset(CamelModel): @@ -137,10 +138,6 @@ def should_add_collection_links(self) -> bool: def should_add_item_links(self) -> bool: return self.create_links and (not self.hidden) - class Config: - json_loads = orjson.loads - json_dumps = orjson_dumps - class Mosaics(CamelModel): """ @@ -187,11 +184,11 @@ class LegendConfig(CamelModel): showing legend labels as scaled values. """ - type: Optional[str] - labels: Optional[List[str]] - trim_start: Optional[int] - trim_end: Optional[int] - scale_factor: Optional[float] + type: Optional[str] = None + labels: Optional[List[str]] = None + trim_start: Optional[int] = None + trim_end: Optional[int] = None + scale_factor: Optional[float] = None class VectorTileOptions(CamelModel): @@ -216,10 +213,10 @@ class VectorTileOptions(CamelModel): tilejson_key: str source_layer: str - fill_color: Optional[str] - stroke_color: Optional[str] - stroke_width: Optional[int] - filter: Optional[List[Any]] + fill_color: Optional[str] = None + stroke_color: Optional[str] = None + stroke_width: Optional[int] = None + filter: Optional[List[Any]] = None class RenderOptionCondition(CamelModel): @@ -329,10 +326,6 @@ class CollectionConfig(BaseModel): render_config: DefaultRenderConfig mosaic_info: MosaicInfo - class Config: - json_loads = orjson.loads - json_dumps = orjson_dumps - class CollectionConfigTable(ModelTableService[CollectionConfig]): _model = CollectionConfig diff --git a/pccommon/pccommon/config/containers.py b/pccommon/pccommon/config/containers.py index a40875b2..045640a1 100644 --- a/pccommon/pccommon/config/containers.py +++ b/pccommon/pccommon/config/containers.py @@ -1,18 +1,20 @@ from typing import Optional -import orjson from pydantic import BaseModel from pccommon.tables import ModelTableService -from pccommon.utils import orjson_dumps class ContainerConfig(BaseModel): has_cdn: bool = False - class Config: - json_loads = orjson.loads - json_dumps = orjson_dumps + # json_loads/json_dumps config have been removed + # the authors seems to indicate that parsing/serialization + # in Rust (pydantic-core) is fast (but maybe not as fast as orjson) + # https://github.com/pydantic/pydantic/discussions/6388 + # class Config: + # json_loads = orjson.loads + # json_dumps = orjson_dumps class ContainerConfigTable(ModelTableService[ContainerConfig]): diff --git a/pccommon/pccommon/config/core.py b/pccommon/pccommon/config/core.py index 75162b2c..8350ece3 100644 --- a/pccommon/pccommon/config/core.py +++ b/pccommon/pccommon/config/core.py @@ -4,8 +4,8 @@ from cachetools import Cache, LRUCache, cachedmethod from cachetools.func import lru_cache from cachetools.keys import hashkey -from pydantic import BaseModel, BaseSettings, Field, PrivateAttr - +from pydantic import BaseModel, Field, PrivateAttr, validator +from pydantic_settings import BaseSettings from pccommon.config.collections import CollectionConfigTable from pccommon.config.containers import ContainerConfigTable from pccommon.constants import DEFAULT_TTL @@ -20,17 +20,26 @@ class TableConfig(BaseModel): account_name: str - account_key: str table_name: str account_url: Optional[str] = None + @validator("account_url") + def validate_url(cls, value: str) -> str: + if value and not value.startswith("http://azurite:"): + raise ValueError( + "Non-azurite account url provided. " + "Account keys can only be used with Azurite emulator." + ) + + return value + class PCAPIsConfig(BaseSettings): _cache: Cache = PrivateAttr(default_factory=lambda: LRUCache(maxsize=10)) app_insights_instrumentation_key: Optional[str] = Field( # type: ignore default=None, - env=APP_INSIGHTS_INSTRUMENTATION_KEY, + json_schema_extra={"env": APP_INSIGHTS_INSTRUMENTATION_KEY}, ) collection_config: TableConfig container_config: TableConfig @@ -46,32 +55,38 @@ class PCAPIsConfig(BaseSettings): debug: bool = False + model_config = { + "env_prefix": ENV_VAR_PCAPIS_PREFIX, + "env_nested_delimiter": "__", + # Mypi is complaining about this with + # error: Incompatible types (expression has type "str", + # TypedDict item "extra" has type "Extra") + "extra": "ignore", # type: ignore + } + @cachedmethod(cache=lambda self: self._cache, key=lambda _: hashkey("collection")) def get_collection_config_table(self) -> CollectionConfigTable: - return CollectionConfigTable.from_account_key( + return CollectionConfigTable.from_environment( account_url=self.collection_config.account_url, account_name=self.collection_config.account_name, - account_key=self.collection_config.account_key, table_name=self.collection_config.table_name, ttl=self.table_value_ttl, ) @cachedmethod(cache=lambda self: self._cache, key=lambda _: hashkey("container")) def get_container_config_table(self) -> ContainerConfigTable: - return ContainerConfigTable.from_account_key( + return ContainerConfigTable.from_environment( account_url=self.container_config.account_url, account_name=self.container_config.account_name, - account_key=self.container_config.account_key, table_name=self.container_config.table_name, ttl=self.table_value_ttl, ) @cachedmethod(cache=lambda self: self._cache, key=lambda _: hashkey("ip_whitelist")) def get_ip_exception_list_table(self) -> IPExceptionListTable: - return IPExceptionListTable.from_account_key( + return IPExceptionListTable.from_environment( account_url=self.ip_exception_config.account_url, account_name=self.ip_exception_config.account_name, - account_key=self.ip_exception_config.account_key, table_name=self.ip_exception_config.table_name, ttl=self.table_value_ttl, ) @@ -80,8 +95,3 @@ def get_ip_exception_list_table(self) -> IPExceptionListTable: @lru_cache(maxsize=1) def from_environment(cls) -> "PCAPIsConfig": return PCAPIsConfig() # type: ignore - - class Config: - env_prefix = ENV_VAR_PCAPIS_PREFIX - extra = "ignore" - env_nested_delimiter = "__" diff --git a/pccommon/pccommon/constants.py b/pccommon/pccommon/constants.py index 8929cbf9..2a8a8d36 100644 --- a/pccommon/pccommon/constants.py +++ b/pccommon/pccommon/constants.py @@ -30,3 +30,11 @@ HTTP_URL = COMMON_ATTRIBUTES["HTTP_URL"] HTTP_STATUS_CODE = COMMON_ATTRIBUTES["HTTP_STATUS_CODE"] HTTP_METHOD = COMMON_ATTRIBUTES["HTTP_METHOD"] + +# This is the Azurite storage account key. +# This is not a key for a real Storage Account and is publicly accessible +# on Azurite's GitHub repo. This is used only in development. +AZURITE_ACCOUNT_KEY = ( + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUz" + "FT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" +) diff --git a/pccommon/pccommon/tables.py b/pccommon/pccommon/tables.py index af60b7b1..006bb6a1 100644 --- a/pccommon/pccommon/tables.py +++ b/pccommon/pccommon/tables.py @@ -1,3 +1,4 @@ +import os from threading import Lock from typing import ( Any, @@ -10,17 +11,20 @@ Tuple, Type, TypeVar, + Union, ) import orjson -from azure.core.credentials import AzureNamedKeyCredential, AzureSasCredential +from azure.core.credentials import AzureNamedKeyCredential from azure.core.exceptions import ResourceNotFoundError from azure.data.tables import TableClient, TableEntity, TableServiceClient +from azure.identity import AzureCliCredential, ManagedIdentityCredential from cachetools import Cache, TTLCache, cachedmethod from cachetools.keys import hashkey from pydantic import BaseModel from pccommon.constants import ( + AZURITE_ACCOUNT_KEY, DEFAULT_IP_EXCEPTIONS_TTL, DEFAULT_TTL, IP_EXCEPTION_PARTITION_KEY, @@ -41,8 +45,10 @@ class TableError(Exception): pass +# TODO: mypy is complaining locally about +# "BaseModel" has no attribute "model_dump_json" def encode_model(m: BaseModel) -> str: - return m.json() + return m.model_dump_json() # type: ignore def decode_dict(s: str) -> Dict[str, Any]: @@ -81,60 +87,49 @@ def __exit__(self, *args: Any) -> None: self._service_client = None @classmethod - def from_sas_token( - cls: Type[T], account_url: str, sas_token: str, table_name: str - ) -> T: - def _get_clients( - _url: str = account_url, _token: str = sas_token, _table: str = table_name - ) -> Tuple[Optional[TableServiceClient], TableClient]: - table_service_client = TableServiceClient( - endpoint=_url, - credential=AzureSasCredential(_token), - ) - return ( - table_service_client, - table_service_client.get_table_client(table_name=_table), - ) - - return cls(_get_clients) - - @classmethod - def from_connection_string( - cls: Type[T], connection_string: str, table_name: str - ) -> T: - def _get_clients( - _conn_str: str = connection_string, _table: str = table_name - ) -> Tuple[Optional[TableServiceClient], TableClient]: - table_service_client = TableServiceClient.from_connection_string( - conn_str=_conn_str - ) - return ( - table_service_client, - table_service_client.get_table_client(table_name=_table), - ) - - return cls(_get_clients) - - @classmethod - def from_account_key( + def from_environment( cls: Type[T], account_name: str, - account_key: str, table_name: str, account_url: Optional[str] = None, ttl: Optional[int] = None, ) -> T: def _get_clients( - _name: str = account_name, - _key: str = account_key, - _url: Optional[str] = account_url, + _account: str = account_name, _table: str = table_name, + _url: Optional[str] = account_url, ) -> Tuple[Optional[TableServiceClient], TableClient]: - _url = _url or f"https://{_name}.table.core.windows.net" - credential = AzureNamedKeyCredential(name=_name, key=_key) + credential: Union[ + AzureNamedKeyCredential, ManagedIdentityCredential, AzureCliCredential + ] + + # Check if the environment is configured to use Azurite and use that key. + # Otherwise, we must use a workload identity. + if _url: + if not _url.startswith("http://azurite:"): + raise ValueError( + "Non-azurite account url provided. " + "Account keys can only be used with Azurite emulator." + ) + + url = _url + credential = AzureNamedKeyCredential( + name=_account, key=AZURITE_ACCOUNT_KEY + ) + else: + client_id = os.environ.get("AZURE_CLIENT_ID") + credential = ( + ManagedIdentityCredential(client_id=client_id) + if client_id + else AzureCliCredential() + ) + + url = f"https://{_account}.table.core.windows.net" + table_service_client = TableServiceClient( - endpoint=_url, credential=credential + endpoint=url, credential=credential ) + return ( table_service_client, table_service_client.get_table_client(table_name=_table), diff --git a/pccommon/pccommon/tracing.py b/pccommon/pccommon/tracing.py index 8e28e6c0..bd4cc7e6 100644 --- a/pccommon/pccommon/tracing.py +++ b/pccommon/pccommon/tracing.py @@ -3,7 +3,6 @@ import re from typing import List, Optional, Tuple, Union, cast -import fastapi from fastapi import Request from opencensus.ext.azure.trace_exporter import AzureExporter from opencensus.trace import execution_context @@ -211,7 +210,7 @@ def _iter_cql(cql: dict, property_name: str) -> Optional[Union[str, List[str]]]: return None -def add_stac_attributes_from_search(search_json: str, request: fastapi.Request) -> None: +def add_stac_attributes_from_search(search_json: str, request: Request) -> None: """ Try to add the Collection ID and Item ID from a search to the current span. """ diff --git a/pccommon/requirements.txt b/pccommon/requirements.txt index 14ee4c05..7ef3042d 100644 --- a/pccommon/requirements.txt +++ b/pccommon/requirements.txt @@ -4,11 +4,13 @@ # # pip-compile --extra=server --output-file=pccommon/requirements.txt ./pccommon/setup.py # -anyio==4.3.0 +annotated-types==0.7.0 + # via pydantic +anyio==4.4.0 # via starlette async-timeout==4.0.3 # via redis -azure-core==1.30.1 +azure-core==1.30.2 # via # azure-data-tables # azure-identity @@ -28,27 +30,27 @@ cachetools==5.3.3 # via # google-auth # pccommon (pccommon/setup.py) -certifi==2024.2.2 +certifi==2024.6.2 # via requests cffi==1.16.0 # via cryptography charset-normalizer==3.3.2 # via requests -cryptography==42.0.5 +cryptography==42.0.8 # via # azure-identity # azure-storage-blob # msal # pyjwt -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via anyio -fastapi==0.90.1 +fastapi-slim==0.111.0 # via pccommon (pccommon/setup.py) -google-api-core==2.18.0 +google-api-core==2.19.0 # via opencensus -google-auth==2.29.0 +google-auth==2.30.0 # via google-api-core -googleapis-common-protos==1.63.0 +googleapis-common-protos==1.63.1 # via google-api-core html-sanitizer==2.4.4 # via pccommon (pccommon/setup.py) @@ -62,7 +64,7 @@ isodate==0.6.1 # via # azure-data-tables # azure-storage-blob -lxml==5.2.1 +lxml==5.2.2 # via # html-sanitizer # lxml-html-clean @@ -70,11 +72,11 @@ lxml-html-clean==0.1.0 # via # html-sanitizer # pccommon (pccommon/setup.py) -msal==1.28.0 +msal==1.28.1 # via # azure-identity # msal-extensions -msal-extensions==0.3.1 +msal-extensions==1.1.0 # via azure-identity multidict==6.0.5 # via yarl @@ -88,8 +90,10 @@ opencensus-ext-azure==1.1.13 # via pccommon (pccommon/setup.py) opencensus-ext-logging==0.1.1 # via pccommon (pccommon/setup.py) -orjson==3.10.4 +orjson==3.10.5 # via pccommon (pccommon/setup.py) +packaging==24.1 + # via msal-extensions portalocker==2.8.2 # via msal-extensions proto-plus==1.23.0 @@ -101,22 +105,29 @@ protobuf==4.25.3 # proto-plus psutil==5.9.8 # via opencensus-ext-azure -pyasn1==0.5.1 +pyasn1==0.6.0 # via # pyasn1-modules # rsa -pyasn1-modules==0.3.0 +pyasn1-modules==0.4.0 # via google-auth -pycparser==2.21 +pycparser==2.22 # via cffi -pydantic==1.10.14 +pydantic==2.7.4 # via - # fastapi + # fastapi-slim # pccommon (pccommon/setup.py) + # pydantic-settings +pydantic-core==2.18.4 + # via pydantic +pydantic-settings==2.3.3 + # via pccommon (pccommon/setup.py) pyhumps==3.5.3 # via pccommon (pccommon/setup.py) pyjwt[crypto]==2.8.0 # via msal +python-dotenv==1.0.1 + # via pydantic-settings redis==4.6.0 # via pccommon (pccommon/setup.py) requests==2.32.3 @@ -137,21 +148,23 @@ sniffio==1.3.1 # via anyio soupsieve==2.5 # via beautifulsoup4 -starlette==0.22.0 +starlette==0.37.2 # via - # fastapi + # fastapi-slim # pccommon (pccommon/setup.py) types-cachetools==4.2.9 # via pccommon (pccommon/setup.py) -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via # anyio # azure-core # azure-data-tables # azure-storage-blob + # fastapi-slim # pydantic + # pydantic-core # starlette -urllib3==2.2.1 +urllib3==2.2.2 # via # pccommon (pccommon/setup.py) # requests diff --git a/pccommon/setup.py b/pccommon/setup.py index 71d596ea..01a5ad63 100644 --- a/pccommon/setup.py +++ b/pccommon/setup.py @@ -4,15 +4,16 @@ # Runtime requirements. inst_reqs = [ - "fastapi==0.90.1", - "starlette>=0.22.0,<0.23.0", + "fastapi-slim==0.111.0", + "starlette>=0.37.2,<0.38.0", "opencensus-ext-azure==1.1.13", "opencensus-ext-logging==0.1.1", "orjson>=3.10.4", "azure-identity==1.16.1", "azure-data-tables==12.5.0", "azure-storage-blob>=12.20.0", - "pydantic>=1.10, <2.0.0", + "pydantic>=2.7,<2.8.0", + "pydantic-settings>=2.3,<2.4", "cachetools~=5.3", "types-cachetools==4.2.9", "pyhumps==3.5.3", diff --git a/pccommon/tests/config/test_mosaic_info.py b/pccommon/tests/config/test_mosaic_info.py index c3c427e1..3eebdcdf 100644 --- a/pccommon/tests/config/test_mosaic_info.py +++ b/pccommon/tests/config/test_mosaic_info.py @@ -38,8 +38,8 @@ def test_parse() -> None: ], "defaultLocation": {"zoom": 8, "coordinates": [47.1113, -120.8578]}, } - model = MosaicInfo.parse_obj(d) - serialized = model.dict(by_alias=True, exclude_unset=True) + model = MosaicInfo.model_validate(d) + serialized = model.model_dump(by_alias=True, exclude_unset=True) assert d == serialized @@ -122,7 +122,7 @@ def test_parse_with_legend() -> None: "defaultLocation": {"zoom": 10, "coordinates": [24.21647, 91.015209]}, } - model = MosaicInfo.parse_obj(d) - serialized = model.dict(by_alias=True, exclude_unset=True) + model = MosaicInfo.model_validate(d) + serialized = model.model_dump(by_alias=True, exclude_unset=True) assert d == serialized diff --git a/pccommon/tests/config/test_render_config.py b/pccommon/tests/config/test_render_config.py index 6cd34240..43ee6698 100644 --- a/pccommon/tests/config/test_render_config.py +++ b/pccommon/tests/config/test_render_config.py @@ -74,12 +74,12 @@ def test_get_render_config() -> None: def test_render_config_parse_max_items() -> None: config = { - "render_params": [], + "render_params": {}, "minzoom": 8, "max_items_per_tile": 10, } - parsed = DefaultRenderConfig.parse_obj(config) + parsed = DefaultRenderConfig.model_validate(config) assert parsed.max_items_per_tile == config["max_items_per_tile"] diff --git a/pccommon/tests/config/test_table_settings.py b/pccommon/tests/config/test_table_settings.py new file mode 100644 index 00000000..bdb765c9 --- /dev/null +++ b/pccommon/tests/config/test_table_settings.py @@ -0,0 +1,22 @@ +import pytest + +from pccommon.config.core import TableConfig + + +def test_raises_on_non_azurite_account_url() -> None: + + invalid_url = "https://example.com" + with pytest.raises(ValueError) as exc_info: + TableConfig(account_url=invalid_url, table_name="test", account_name="test") + + assert ( + "Non-azurite account url provided. " + "Account keys can only be used with Azurite emulator." + ) in str(exc_info.value) + + +def test_settings_accepts_azurite_url() -> None: + valid_url = "http://azurite:12345" + + config = TableConfig(account_url=valid_url, table_name="test", account_name="test") + assert config.account_url == valid_url diff --git a/pccommon/tests/test_timeouts.py b/pccommon/tests/test_timeouts.py index b32ce056..da54d8cb 100644 --- a/pccommon/tests/test_timeouts.py +++ b/pccommon/tests/test_timeouts.py @@ -4,8 +4,7 @@ import pytest from fastapi import FastAPI -# from fastapi.responses import PlainTextResponse -from httpx import AsyncClient +from httpx import ASGITransport, AsyncClient from starlette.status import HTTP_504_GATEWAY_TIMEOUT from pccommon.middleware import add_timeout @@ -13,29 +12,28 @@ TIMEOUT_SECONDS = 2 BASE_URL = "http://test" -# Setup test app and endpoints to test middleware on -# ================================== -app = FastAPI() -app.state.service_name = "test" - - -@app.get("/asleep") -async def asleep() -> Any: - await asyncio.sleep(1) - return {} - - -# Run this after registering the routes +@pytest.mark.asyncio +async def test_add_timeout() -> None: -add_timeout(app, timeout_seconds=0.001) + # Setup test app and endpoints to test middleware on + # ================================== + app = FastAPI() + app.state.service_name = "test" -@pytest.mark.asyncio -async def test_add_timeout() -> None: + @app.get("/asleep") + async def asleep() -> Any: + await asyncio.sleep(1) + return {} - client = AsyncClient(app=app, base_url=BASE_URL) + # Run this after registering the routes + add_timeout(app, timeout_seconds=0.001) - response = await client.get("/asleep") + async with AsyncClient( + transport=ASGITransport(app=app), # type: ignore + base_url=BASE_URL, + ) as client: + response = await client.get("/asleep") assert response.status_code == HTTP_504_GATEWAY_TIMEOUT diff --git a/pcfuncs/Dockerfile b/pcfuncs/Dockerfile index 3beb133f..888f90ce 100644 --- a/pcfuncs/Dockerfile +++ b/pcfuncs/Dockerfile @@ -1,7 +1,8 @@ -FROM mcr.microsoft.com/azure-functions/python:4-python3.8 +FROM mcr.microsoft.com/azure-functions/python:4-python3.10 # git required for pip installs from git RUN apt update && apt install -y git +RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash ENV AzureWebJobsScriptRoot=/home/site/wwwroot \ AzureFunctionsJobHost__Logging__Console__IsEnabled=true diff --git a/pcfuncs/animation/models.py b/pcfuncs/animation/models.py index 28b8a683..39290cda 100644 --- a/pcfuncs/animation/models.py +++ b/pcfuncs/animation/models.py @@ -3,7 +3,7 @@ from dateutil.relativedelta import relativedelta from funclib.models import RenderOptions -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, Field, field_validator from .constants import MAX_FRAMES @@ -44,12 +44,12 @@ class AnimationRequest(BaseModel): data_api_url: Optional[str] = None """Override for the data API URL. Useful for testing.""" - @validator("render_params") + @field_validator("render_params") def _validate_render_params(cls, v: str) -> str: RenderOptions.from_query_params(v) return v - @validator("unit") + @field_validator("unit") def _validate_unit(cls, v: str) -> str: if v not in _deltas: raise ValueError( diff --git a/pcfuncs/animation/settings.py b/pcfuncs/animation/settings.py index e9d99d00..7edd2aa4 100644 --- a/pcfuncs/animation/settings.py +++ b/pcfuncs/animation/settings.py @@ -18,9 +18,10 @@ class AnimationSettings(BaseExporterSettings): output_storage_url: str = DEFAULT_ANIMATION_CONTAINER_URL tile_request_concurrency: int = DEFAULT_CONCURRENCY - class Config: - env_prefix = ANIMATION_SETTINGS_PREFIX - env_nested_delimiter = "__" + model_config = { + "env_prefix": ANIMATION_SETTINGS_PREFIX, + "env_nested_delimiter": "__", # type: ignore + } @classmethod @cachedmethod(lambda cls: cls._cache) diff --git a/pcfuncs/funclib/models.py b/pcfuncs/funclib/models.py index 9afeaeba..978d45c7 100644 --- a/pcfuncs/funclib/models.py +++ b/pcfuncs/funclib/models.py @@ -78,7 +78,7 @@ class RenderOptions(BaseModel): @property def encoded_query_string(self) -> str: - options = self.dict( + options = self.model_dump( exclude_defaults=True, exclude_none=True, exclude_unset=True ) encoded_options: List[str] = [] diff --git a/pcfuncs/funclib/settings.py b/pcfuncs/funclib/settings.py index eb049f0a..c34136a1 100644 --- a/pcfuncs/funclib/settings.py +++ b/pcfuncs/funclib/settings.py @@ -2,23 +2,18 @@ from typing import Optional from azure.storage.blob import ContainerClient -from pydantic import BaseSettings +from pydantic_settings import BaseSettings from pccommon.blob import get_container_client class BaseExporterSettings(BaseSettings): api_root_url: str = "https://planetarycomputer.microsoft.com/api/data/v1" - output_storage_url: str - output_sas: Optional[str] = None - output_account_key: Optional[str] = None def get_container_client(self) -> ContainerClient: return get_container_client( self.output_storage_url, - sas_token=self.output_sas, - account_key=self.output_account_key, ) def get_register_url(self, data_api_url_override: Optional[str] = None) -> str: diff --git a/pcfuncs/image/__init__.py b/pcfuncs/image/__init__.py index b18c933b..0c692db8 100644 --- a/pcfuncs/image/__init__.py +++ b/pcfuncs/image/__init__.py @@ -9,7 +9,7 @@ from pydantic import ValidationError from .models import ImageRequest, ImageResponse -from .settings import ImageSettings +from .settings import get_settings from .utils import get_min_zoom, upload_image logger = logging.getLogger(__name__) @@ -62,7 +62,7 @@ async def main(req: func.HttpRequest) -> func.HttpResponse: async def handle_request(req: ImageRequest) -> ImageResponse: - settings = ImageSettings.get() + settings = get_settings() geom = req.get_geometry() bbox = Bbox.from_geom(geom) render_options = req.get_render_options() diff --git a/pcfuncs/image/models.py b/pcfuncs/image/models.py index 7079ff16..cce81e3e 100644 --- a/pcfuncs/image/models.py +++ b/pcfuncs/image/models.py @@ -2,9 +2,9 @@ from funclib.models import RenderOptions from funclib.raster import ExportFormats -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, Field, field_validator, ValidationInfo -from .settings import ImageSettings +from .settings import get_settings from .utils import get_geom_from_cql @@ -53,12 +53,14 @@ def get_geometry(self) -> Dict[str, Any]: def get_render_options(self) -> RenderOptions: return RenderOptions.from_query_params(self.render_params) - @validator("geometry") + @field_validator("geometry") def _validate_cql( - cls, v: Optional[Dict[str, Any]], values: Dict[str, Any] + cls, + v: Optional[Dict[str, Any]], + info: ValidationInfo, ) -> Dict[str, Any]: if not v: - cql = values["cql"] + cql = info.data["cql"] v = get_geom_from_cql(cql) if not v: raise ValueError( @@ -67,15 +69,15 @@ def _validate_cql( ) return v - @validator("render_params") + @field_validator("render_params") def _validate_render_params(cls, v: str) -> str: RenderOptions.from_query_params(v) return v - @validator("rows") - def _validate_rows(cls, v: int, values: Dict[str, Any]) -> int: - settings = ImageSettings.get() - cols = int(values["cols"]) + @field_validator("rows") + def _validate_rows(cls, v: int, info: ValidationInfo) -> int: + settings = get_settings() + cols = int(info.data["cols"]) if cols * v > settings.max_pixels: raise ValueError( f"Too many pixels requested: {cols * v} > {settings.max_pixels}. " @@ -83,10 +85,10 @@ def _validate_rows(cls, v: int, values: Dict[str, Any]) -> int: ) return v - @validator("show_branding") - def _validate_show_branding(cls, v: bool, values: Dict[str, Any]) -> bool: + @field_validator("show_branding") + def _validate_show_branding(cls, v: bool, info: ValidationInfo) -> bool: if v: - if values["format"] != ExportFormats.PNG: + if info.data["format"] != ExportFormats.PNG: raise ValueError("Branding is only supported for PNG images.") return v diff --git a/pcfuncs/image/settings.py b/pcfuncs/image/settings.py index 0a2c6526..24922363 100644 --- a/pcfuncs/image/settings.py +++ b/pcfuncs/image/settings.py @@ -1,12 +1,8 @@ import logging -import os -from typing import Optional -from azure.storage.blob import ContainerClient -from cachetools import Cache, LRUCache, cachedmethod +from cachetools import LRUCache, cached from funclib.settings import BaseExporterSettings -from pccommon.blob import get_container_client IMAGE_SETTINGS_PREFIX = "IMAGE_" DEFAULT_CONCURRENCY = 10 @@ -15,7 +11,7 @@ class ImageSettings(BaseExporterSettings): - _cache: Cache = LRUCache(maxsize=100) + # _cache: Cache = LRUCache(maxsize=100) tile_request_concurrency: int = DEFAULT_CONCURRENCY @@ -23,34 +19,23 @@ class ImageSettings(BaseExporterSettings): max_tile_count: int = 144 max_pixels: int = 144 * 512 * 512 - def get_container_client(self) -> ContainerClient: - return get_container_client( - self.output_storage_url, - sas_token=self.output_sas, - account_key=self.output_account_key, - ) - - def get_register_url(self, data_api_url_override: Optional[str] = None) -> str: - return os.path.join( - data_api_url_override or self.api_root_url, "mosaic/register/" - ) - - def get_mosaic_info_url( - self, collection_id: str, data_api_url_override: Optional[str] = None - ) -> str: - return os.path.join( - data_api_url_override or self.api_root_url, - f"mosaic/info?collection={collection_id}", - ) - - class Config: - env_prefix = IMAGE_SETTINGS_PREFIX - env_nested_delimiter = "__" - - @classmethod - @cachedmethod(lambda cls: cls._cache) - def get(cls) -> "ImageSettings": - settings = ImageSettings() # type: ignore - logger.info(f"API URL: {settings.api_root_url}") - logger.info(f"Concurrency limit: {settings.tile_request_concurrency}") - return settings + model_config = { + "env_prefix": IMAGE_SETTINGS_PREFIX, + "env_nested_delimiter": "__", # type: ignore + } + + # @classmethod + # @cachedmethod(lambda cls: cls._cache) + # def get(cls) -> "ImageSettings": + # settings = ImageSettings() # type: ignore + # logger.info(f"API URL: {settings.api_root_url}") + # logger.info(f"Concurrency limit: {settings.tile_request_concurrency}") + # return settings + + +@cached(LRUCache(maxsize=100)) # type: ignore +def get_settings() -> ImageSettings: + settings = ImageSettings() # type: ignore + logger.info(f"API URL: {settings.api_root_url}") + logger.info(f"Concurrency limit: {settings.tile_request_concurrency}") + return settings diff --git a/pcfuncs/image/utils.py b/pcfuncs/image/utils.py index 9680cd13..f1e027b5 100644 --- a/pcfuncs/image/utils.py +++ b/pcfuncs/image/utils.py @@ -7,13 +7,13 @@ import aiohttp from funclib.models import RenderOptions -from .settings import ImageSettings +from .settings import get_settings async def get_min_zoom( collection_id: str, data_api_url_override: Optional[str] = None ) -> Optional[int]: - settings = ImageSettings.get() + settings = get_settings() async with aiohttp.ClientSession() as session: resp = await session.get( settings.get_mosaic_info_url(collection_id, data_api_url_override) @@ -32,7 +32,7 @@ async def get_min_zoom( def upload_image(gif: io.BytesIO, collection_name: str) -> str: - settings = ImageSettings.get() + settings = get_settings() filename = f"mspc-{collection_name}-{uuid4().hex}.png" blob_url = os.path.join(settings.output_storage_url, filename) with settings.get_container_client() as container_client: @@ -108,7 +108,7 @@ async def register_search_and_get_tile_url( render_options: RenderOptions, data_api_url_override: Optional[str] = None, ) -> str: - settings = ImageSettings.get() + settings = get_settings() register_url = settings.get_register_url(data_api_url_override) async with aiohttp.ClientSession() as session: diff --git a/pcfuncs/ipban/__init__.py b/pcfuncs/ipban/__init__.py new file mode 100644 index 00000000..72f0e645 --- /dev/null +++ b/pcfuncs/ipban/__init__.py @@ -0,0 +1,29 @@ +import datetime +import logging + +import azure.functions as func +from azure.data.tables import TableServiceClient +from azure.identity import DefaultAzureCredential +from azure.monitor.query import LogsQueryClient + +from .config import settings +from .models import UpdateBannedIPTask + +logger = logging.getLogger(__name__) + + +def main(mytimer: func.TimerRequest) -> None: + utc_timestamp: str = ( + datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat() + ) + logger.info("Updating the ip ban list at %s", utc_timestamp) + credential: DefaultAzureCredential = DefaultAzureCredential() + with LogsQueryClient(credential) as logs_query_client: + with TableServiceClient( + endpoint=settings.storage_account_url, credential=credential + ) as table_service_client: + with table_service_client.create_table_if_not_exists( + settings.banned_ip_table + ) as table_client: + task = UpdateBannedIPTask(logs_query_client, table_client) + task.run() diff --git a/pcfuncs/ipban/config.py b/pcfuncs/ipban/config.py new file mode 100644 index 00000000..1fee8478 --- /dev/null +++ b/pcfuncs/ipban/config.py @@ -0,0 +1,18 @@ +# config.py +from pydantic import Field +from pydantic_settings import BaseSettings + +class Settings(BaseSettings): + storage_account_url: str = Field(json_schema_extra={"env": "STORAGE_ACCOUNT_URL"}) + banned_ip_table: str = Field(json_schema_extra={"env": "BANNED_IP_TABLE"}) + log_analytics_workspace_id: str = Field(json_schema_extra={"env": "LOG_ANALYTICS_WORKSPACE_ID"}) + + # Time and threshold settings + time_window_in_hours: int = Field(default=24, json_schema_extra={"env": "TIME_WINDOW_IN_HOURS"}) + threshold_read_count_in_gb: int = Field( + default=5120, json_schema_extra={"env": "THRESHOLD_READ_COUNT_IN_GB"} + ) + + +# Create a global settings instance +settings = Settings() # type: ignore diff --git a/pcfuncs/ipban/function.json b/pcfuncs/ipban/function.json new file mode 100644 index 00000000..2b55fa8e --- /dev/null +++ b/pcfuncs/ipban/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "name": "mytimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "0 */1 * * *" + } + ] +} diff --git a/pcfuncs/ipban/models.py b/pcfuncs/ipban/models.py new file mode 100644 index 00000000..71ec5021 --- /dev/null +++ b/pcfuncs/ipban/models.py @@ -0,0 +1,70 @@ +import logging +from typing import Any, List, Set + +from azure.data.tables import TableClient, UpdateMode +from azure.monitor.query import LogsQueryClient +from azure.monitor.query._models import LogsTableRow + +from .config import settings + + +class UpdateBannedIPTask: + def __init__( + self, + logs_query_client: LogsQueryClient, + table_client: TableClient, + ) -> None: + self.log_query_client = logs_query_client + self.table_client = table_client + + def run(self) -> List[LogsTableRow]: + query_result: List[LogsTableRow] = self.get_blob_logs_query_result() + logging.info(f"Kusto query result: {query_result}") + self.update_banned_ips(query_result) + return query_result + + def get_blob_logs_query_result(self) -> List[LogsTableRow]: + query: str = f""" + StorageBlobLogs + | where TimeGenerated > ago({settings.time_window_in_hours}h) + | extend IpAddress = tostring(split(CallerIpAddress, ":")[0]) + | where OperationName == 'GetBlob' + | where not(ipv4_is_private(IpAddress)) + | summarize readcount = sum(ResponseBodySize) / (1024 * 1024 * 1024) + by IpAddress + | where readcount > {settings.threshold_read_count_in_gb} + """ + response: Any = self.log_query_client.query_workspace( + settings.log_analytics_workspace_id, query, timespan=None + ) + return response.tables[0].rows + + def update_banned_ips(self, query_result: List[LogsTableRow]) -> None: + existing_ips = { + entity["RowKey"] for entity in self.table_client.list_entities() + } + result_ips: Set[str] = set() + for result in query_result: + ip_address: str = result[0] + read_count: int = int(result[1]) + result_ips.add(ip_address) + entity = { + "PartitionKey": ip_address, + "RowKey": ip_address, + "ReadCount": read_count, + "Threshold": settings.threshold_read_count_in_gb, + "TimeWindow": settings.time_window_in_hours, + } + + if ip_address in existing_ips: + self.table_client.update_entity(entity, mode=UpdateMode.REPLACE) + else: + self.table_client.create_entity(entity) + + for ip_address in existing_ips: + if ip_address not in result_ips: + self.table_client.delete_entity( + partition_key=ip_address, row_key=ip_address + ) + + logging.info("IP ban list has been updated successfully") diff --git a/pcfuncs/requirements-deploy.txt b/pcfuncs/requirements-deploy.txt index 3c95a933..0dc9275c 100644 --- a/pcfuncs/requirements-deploy.txt +++ b/pcfuncs/requirements-deploy.txt @@ -12,9 +12,10 @@ dateutils==0.6.12 mercantile==1.2.1 pillow==10.3.0 pyproj==3.3.1 -pydantic>=1.9,<2.0.0 +pydantic>=2.7,<2.8 rasterio==1.3.* - +azure-monitor-query==1.3.0 +pytest-mock==3.14.0 # The deploy process needs symlinks to bring in # pctasks libraries. Symlink is created in deploy script ./pccommon_linked diff --git a/pcfuncs/requirements.txt b/pcfuncs/requirements.txt index b1cf10b3..f0053bf0 100644 --- a/pcfuncs/requirements.txt +++ b/pcfuncs/requirements.txt @@ -12,9 +12,10 @@ dateutils==0.6.12 mercantile==1.2.1 pillow==10.3.0 pyproj==3.3.1 -pydantic>=1.9,<2.0.0 +pydantic>=2.7,<2.8 rasterio==1.3.* - +azure-monitor-query==1.3.0 +pytest-mock==3.14.0 # Deployment needs to copy the local code into # the app code directory, so requires a separate # requirements file. diff --git a/pcfuncs/tests/conftest.py b/pcfuncs/tests/conftest.py new file mode 100644 index 00000000..398fc9b2 --- /dev/null +++ b/pcfuncs/tests/conftest.py @@ -0,0 +1,29 @@ +from typing import List + +import pytest + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--no-integration", + action="store_true", + default=False, + help="don't run integration tests", + ) + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line("markers", "integration: mark as an integration test") + + +def pytest_collection_modifyitems( + config: pytest.Config, items: List[pytest.Item] +) -> None: + if config.getoption("--no-integration"): + # --no-integration given in cli: skip integration tests + skip_integration = pytest.mark.skip( + reason="needs --no-integration option to run" + ) + for item in items: + if "integration" in item.keywords: + item.add_marker(skip_integration) diff --git a/pcfuncs/tests/ipban/__init__.py b/pcfuncs/tests/ipban/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py new file mode 100644 index 00000000..749c4fd7 --- /dev/null +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -0,0 +1,136 @@ +import logging +import uuid +from typing import Any, Generator, List, Tuple +from unittest.mock import MagicMock + +import pytest +from azure.data.tables import TableClient, TableServiceClient +from azure.data.tables._entity import TableEntity +from azure.identity import DefaultAzureCredential +from azure.monitor.query import LogsQueryClient +from azure.monitor.query._models import LogsTableRow +from ipban.config import settings +from ipban.models import UpdateBannedIPTask +from pytest_mock import MockerFixture + +MOCK_LOGS_QUERY_RESULT = [("192.168.1.1", 8000), ("192.168.1.4", 12000)] +TEST_ID = str(uuid.uuid4()).replace("-", "") # dash is not allowed in table name +TEST_BANNED_IP_TABLE = f"testblobstoragebannedip{TEST_ID}" + +logger = logging.getLogger(__name__) +PREPOPULATED_ENTITIES = [ + { + "PartitionKey": "192.168.1.1", + "RowKey": "192.168.1.1", + "ReadCount": 647, + "Threshold": settings.threshold_read_count_in_gb, + "TimeWindow": settings.time_window_in_hours, + }, + { + "PartitionKey": "192.168.1.2", + "RowKey": "192.168.1.2", + "ReadCount": 214, + "Threshold": settings.threshold_read_count_in_gb, + "TimeWindow": settings.time_window_in_hours, + }, + { + "PartitionKey": "192.168.1.3", + "RowKey": "192.168.1.3", + "ReadCount": 550, + "Threshold": settings.threshold_read_count_in_gb, + "TimeWindow": settings.time_window_in_hours, + }, +] + + +def populate_banned_ip_table(table_client: TableClient) -> None: + for entity in PREPOPULATED_ENTITIES: + table_client.create_entity(entity) + + +@pytest.fixture +def mock_clients( + mocker: MockerFixture, +) -> Generator[Tuple[MagicMock, TableClient], Any, None]: + mock_response: MagicMock = mocker.MagicMock() + mock_response.tables[0].rows = MOCK_LOGS_QUERY_RESULT + logs_query_client: MagicMock = mocker.MagicMock() + logs_query_client.query_workspace.return_value = mock_response + CONNECTION_STRING: str = ( + "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" + "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu" + "Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" + "TableEndpoint=http://azurite:10002/devstoreaccount1;" + ) + + # Use Azurite for unit tests and populate the table with initial data + with TableServiceClient.from_connection_string( + CONNECTION_STRING + ) as table_service_client: + with table_service_client.create_table_if_not_exists( + table_name=TEST_BANNED_IP_TABLE + ) as table_client: + + # Pre-populate the banned IP table + populate_banned_ip_table(table_client) + + # Yield the clients for use + yield logs_query_client, table_client + + # Delete the test table + table_service_client.delete_table(TEST_BANNED_IP_TABLE) + + +@pytest.fixture +def integration_clients( + mocker: MockerFixture, +) -> Generator[Tuple[LogsQueryClient, TableClient], Any, None]: + credential: DefaultAzureCredential = DefaultAzureCredential() + with LogsQueryClient(credential) as logs_query_client: + with TableServiceClient( + endpoint=settings.storage_account_url, credential=credential + ) as table_service_client: + with table_service_client.create_table_if_not_exists( + TEST_BANNED_IP_TABLE + ) as table_client: + + # Pre-populate the banned IP table + populate_banned_ip_table(table_client) + + # Yield the clients for use + yield logs_query_client, table_client + + # Delete the test table + table_service_client.delete_table(TEST_BANNED_IP_TABLE) + + +@pytest.mark.integration +def test_update_banned_ip_integration( + integration_clients: Tuple[LogsQueryClient, TableClient] +) -> None: + logger.info(f"Test id: {TEST_ID} - integration test is running") + logs_query_client, table_client = integration_clients + assert len(list(table_client.list_entities())) == len(PREPOPULATED_ENTITIES) + task: UpdateBannedIPTask = UpdateBannedIPTask(logs_query_client, table_client) + # retrieve the logs query result from pc-api-loganalytics + logs_query_result: List[LogsTableRow] = task.run() + assert len(list(table_client.list_entities())) == len(logs_query_result) + for ip, expected_read_count in logs_query_result: + entity: TableEntity = table_client.get_entity(ip, ip) + assert entity["ReadCount"] == expected_read_count + assert entity["Threshold"] == settings.threshold_read_count_in_gb + assert entity["TimeWindow"] == settings.time_window_in_hours + + +def test_update_banned_ip(mock_clients: Tuple[MagicMock, TableClient]) -> None: + logger.info(f"Test id: {TEST_ID} - unit test is running") + mock_logs_query_client, table_client = mock_clients + assert len(list(table_client.list_entities())) == len(PREPOPULATED_ENTITIES) + task: UpdateBannedIPTask = UpdateBannedIPTask(mock_logs_query_client, table_client) + task.run() + assert len(list(table_client.list_entities())) == len(MOCK_LOGS_QUERY_RESULT) + for ip, expected_read_count in MOCK_LOGS_QUERY_RESULT: + entity = table_client.get_entity(ip, ip) + assert entity["ReadCount"] == expected_read_count + assert entity["Threshold"] == settings.threshold_read_count_in_gb + assert entity["TimeWindow"] == settings.time_window_in_hours diff --git a/pcstac/pcstac/client.py b/pcstac/pcstac/client.py index 017f6034..3568692f 100644 --- a/pcstac/pcstac/client.py +++ b/pcstac/pcstac/client.py @@ -45,18 +45,6 @@ class PCClient(CoreCrudClient): extra_conformance_classes: List[str] = attr.ib(factory=list) - def conformance_classes(self) -> List[str]: - """Generate conformance classes list.""" - base_conformance_classes = self.base_conformance_classes.copy() - - for extension in self.extensions: - extension_classes = getattr(extension, "conformance_classes", []) - base_conformance_classes.extend(extension_classes) - - base_conformance_classes.extend(self.extra_conformance_classes) - - return sorted(list(set(base_conformance_classes))) - def inject_collection_extras( self, collection: Collection, @@ -227,7 +215,7 @@ async def _fetch() -> ItemCollection: ) return item_collection - search_json = search_request.json() + search_json = search_request.model_dump_json() add_stac_attributes_from_search(search_json, request) logger.info( diff --git a/pcstac/pcstac/config.py b/pcstac/pcstac/config.py index e994042d..78d02fb2 100644 --- a/pcstac/pcstac/config.py +++ b/pcstac/pcstac/config.py @@ -2,7 +2,8 @@ from urllib.parse import urljoin from fastapi import Request -from pydantic import BaseModel, BaseSettings, Field +from pydantic import BaseModel, Field +from pydantic_settings import BaseSettings from stac_fastapi.extensions.core import ( FieldsExtension, FilterExtension, @@ -88,17 +89,38 @@ class Settings(BaseSettings): version of application """ - api = PCAPIsConfig.from_environment() + api: PCAPIsConfig = PCAPIsConfig.from_environment() debug: bool = False - tiler_href: str = Field(env=TILER_HREF_ENV_VAR, default="") - db_max_conn_size: int = Field(env=DB_MAX_CONN_ENV_VAR, default=1) - db_min_conn_size: int = Field(env=DB_MIN_CONN_ENV_VAR, default=1) + tiler_href: str = Field( + default="", + json_schema_extra={"env": TILER_HREF_ENV_VAR}, + ) + db_max_conn_size: int = Field( + default=1, + json_schema_extra={"env": DB_MAX_CONN_ENV_VAR}, + ) + db_min_conn_size: int = Field( + default=1, + json_schema_extra={"env": DB_MIN_CONN_ENV_VAR}, + ) openapi_url: str = "/openapi.json" api_version: str = f"v{API_VERSION}" rate_limits: RateLimits = RateLimits() back_pressures: BackPressures = BackPressures() - request_timeout: int = Field(env=REQUEST_TIMEOUT_ENV_VAR, default=30) + request_timeout: int = Field( + default=30, + json_schema_extra={"env": REQUEST_TIMEOUT_ENV_VAR}, + ) + + model_config = { + "env_prefix": ENV_VAR_PCAPIS_PREFIX, + "env_nested_delimiter": "__", + # Mypi is complaining about this with + # error: Incompatible types (expression has type "str", + # TypedDict item "extra" has type "Extra") + "extra": "ignore", # type: ignore + } def get_tiler_href(self, request: Request) -> str: """Generates the tiler HREF. @@ -113,11 +135,6 @@ def get_tiler_href(self, request: Request) -> str: else: return self.tiler_href - class Config: - env_prefix = ENV_VAR_PCAPIS_PREFIX - extra = "ignore" - env_nested_delimiter = "__" - @lru_cache def get_settings() -> Settings: diff --git a/pcstac/pcstac/filter.py b/pcstac/pcstac/filter.py index 11a9f111..67147bb1 100644 --- a/pcstac/pcstac/filter.py +++ b/pcstac/pcstac/filter.py @@ -2,7 +2,6 @@ from buildpg import render from fastapi import Request -from fastapi.responses import JSONResponse from stac_fastapi.pgstac.extensions.filter import FiltersClient from stac_fastapi.types.errors import NotFoundError @@ -13,7 +12,7 @@ class PCFiltersClient(FiltersClient): async def get_queryables( self, request: Request, collection_id: Optional[str] = None, **kwargs: Any - ) -> JSONResponse: + ) -> Dict[str, Any]: """Override pgstac backend get_queryables to make use of cached results""" async def _fetch() -> Dict: @@ -34,6 +33,4 @@ async def _fetch() -> Dict: return queryables cache_key = f"{CACHE_KEY_QUERYABLES}:{collection_id}" - queryables = await cached_result(_fetch, cache_key, request) - headers = {"Content-Type": "application/schema+json"} - return JSONResponse(queryables, headers=headers) + return await cached_result(_fetch, cache_key, request) diff --git a/pcstac/pcstac/main.py b/pcstac/pcstac/main.py index ad6fc55d..4e47b86d 100644 --- a/pcstac/pcstac/main.py +++ b/pcstac/pcstac/main.py @@ -1,19 +1,28 @@ """FastAPI application using PGStac.""" +from contextlib import asynccontextmanager import logging import os -from typing import Any, Dict +from typing import Any, Dict, AsyncGenerator from fastapi import FastAPI, Request from fastapi.exceptions import RequestValidationError, StarletteHTTPException from fastapi.openapi.utils import get_openapi from fastapi.responses import ORJSONResponse from stac_fastapi.api.errors import DEFAULT_STATUS_CODES -from stac_fastapi.api.models import create_get_request_model, create_post_request_model +from stac_fastapi.api.models import ( + create_get_request_model, + create_post_request_model, + create_request_model, +) +from stac_fastapi.extensions.core import TokenPaginationExtension +from stac_fastapi.api.middleware import ProxyHeaderMiddleware from stac_fastapi.pgstac.config import Settings from stac_fastapi.pgstac.db import close_db_connection, connect_to_db +from starlette.middleware import Middleware from starlette.middleware.cors import CORSMiddleware from starlette.responses import PlainTextResponse +from brotli_asgi import BrotliMiddleware from pccommon.logging import ServiceName, init_logging from pccommon.middleware import TraceMiddleware, add_timeout, http_exception_handler @@ -29,7 +38,7 @@ get_settings, ) from pcstac.errors import PC_DEFAULT_STATUS_CODES -from pcstac.search import PCSearch, PCSearchGetRequest, RedisBaseItemCache +from pcstac.search import PCSearch, PCSearchGetRequest, RedisBaseItemCache, PCItemCollectionUri DEBUG: bool = os.getenv("DEBUG") == "TRUE" or False @@ -46,11 +55,30 @@ app_settings = get_settings() +items_get_request_model = PCItemCollectionUri +if any(isinstance(ext, TokenPaginationExtension) for ext in EXTENSIONS): + items_get_request_model = create_request_model( + model_name="ItemCollectionUri", + base_model=PCItemCollectionUri, + mixins=[TokenPaginationExtension().GET], + request_type="GET", + ) + search_get_request_model = create_get_request_model( EXTENSIONS, base_model=PCSearchGetRequest ) search_post_request_model = create_post_request_model(EXTENSIONS, base_model=PCSearch) + +@asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncGenerator: + """FastAPI Lifespan.""" + await connect_to_db(app) + await connect_to_redis(app) + yield + await close_db_connection(app) + + api = PCStacApi( title=API_TITLE, description=API_DESCRIPTION, @@ -63,11 +91,29 @@ ), client=PCClient.create(post_request_model=search_post_request_model), extensions=EXTENSIONS, - app=FastAPI(root_path=APP_ROOT_PATH, default_response_class=ORJSONResponse), + app=FastAPI( + root_path=APP_ROOT_PATH, + default_response_class=ORJSONResponse, + lifespan=lifespan, + ), + items_get_request_model=items_get_request_model, search_get_request_model=search_get_request_model, search_post_request_model=search_post_request_model, response_class=ORJSONResponse, exceptions={**DEFAULT_STATUS_CODES, **PC_DEFAULT_STATUS_CODES}, + middlewares=[ + Middleware(BrotliMiddleware), + Middleware(ProxyHeaderMiddleware), + Middleware(TraceMiddleware, service_name=ServiceName.STAC), + # Note: If requests are being sent through an application gateway like + # nginx-ingress, you may need to configure CORS through that system. + Middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["GET", "POST"], + allow_headers=["*"], + ), + ], ) app: FastAPI = api.app @@ -76,31 +122,6 @@ add_timeout(app, app_settings.request_timeout) -app.add_middleware(TraceMiddleware, service_name=app.state.service_name) - -# Note: If requests are being sent through an application gateway like -# nginx-ingress, you may need to configure CORS through that system. -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_methods=["GET", "POST"], - allow_headers=["*"], -) - - -@app.on_event("startup") -async def startup_event() -> None: - """Connect to database on startup.""" - await connect_to_db(app) - await connect_to_redis(app) - - -@app.on_event("shutdown") -async def shutdown_event() -> None: - """Close database connection.""" - await close_db_connection(app) - - app.add_exception_handler(Exception, http_exception_handler) diff --git a/pcstac/pcstac/search.py b/pcstac/pcstac/search.py index 931866c7..b40591f9 100644 --- a/pcstac/pcstac/search.py +++ b/pcstac/pcstac/search.py @@ -1,79 +1,43 @@ +import re import logging -from typing import Any, Callable, Coroutine, Dict, List, Optional, Union +from typing import Any, Callable, Coroutine, Dict, Optional import attr -from geojson_pydantic.geometries import ( - GeometryCollection, - LineString, - MultiLineString, - MultiPoint, - MultiPolygon, - Point, - Polygon, -) -from pydantic import validator -from pydantic.types import conint -from pystac.utils import str_to_datetime +from fastapi import Query +from pydantic import Field, field_validator from stac_fastapi.api.models import BaseSearchGetRequest, ItemCollectionUri +from stac_fastapi.types.rfc3339 import str_to_interval, DateTimeType from stac_fastapi.pgstac.types.base_item_cache import BaseItemCache from stac_fastapi.pgstac.types.search import PgstacSearch from starlette.requests import Request +from typing_extensions import Annotated from pccommon.redis import cached_result from pcstac.contants import CACHE_KEY_BASE_ITEM -DEFAULT_LIMIT = 250 +DEFAULT_LIMIT: int = 250 logger = logging.getLogger(__name__) +def _patch_datetime(value: str) -> str: + values = value.split("/") + for ix, v in enumerate(values): + if re.match(r"^(\d\d\d\d)\-(\d\d)\-(\d\d)$", v): + values[ix] = f"{v}T00:00:00Z" + return "/".join(values) + + class PCSearch(PgstacSearch): # Increase the default limit for performance # Ignore "Illegal type annotation: call expression not allowed" - limit: Optional[conint(ge=1, le=1000)] = DEFAULT_LIMIT # type:ignore - - # Can be removed when - # https://github.com/stac-utils/stac-fastapi/issues/187 is closed - intersects: Optional[ - Union[ - Point, - MultiPoint, - LineString, - MultiLineString, - Polygon, - MultiPolygon, - GeometryCollection, - ] - ] - - @validator("datetime") - def validate_datetime(cls, v: str) -> str: - """Validate datetime. - - Custom to allow for users to supply dates only. - """ - if "/" in v: - values = v.split("/") - else: - # Single date is interpreted as end date - values = ["..", v] - - dates: List[str] = [] - for value in values: - if value == "..": - dates.append(value) - continue - - str_to_datetime(value) - dates.append(value) - - if ".." not in dates: - if str_to_datetime(dates[0]) > str_to_datetime(dates[1]): - raise ValueError( - "Invalid datetime range, must match format (begin_date, end_date)" - ) - - return v + limit: Annotated[Optional[int], Field(strict=True, ge=1, le=1000)] = DEFAULT_LIMIT + + @field_validator("datetime", mode="before") + @classmethod + def validate_datetime_before(cls, value: str) -> str: + """Add HH-MM-SS and Z to YYYY-MM-DD datetime.""" + return _patch_datetime(value) class RedisBaseItemCache(BaseItemCache): @@ -106,9 +70,19 @@ async def _fetch() -> Dict[str, Any]: @attr.s class PCItemCollectionUri(ItemCollectionUri): - limit: Optional[int] = attr.ib(default=DEFAULT_LIMIT) # type:ignore + limit: Annotated[Optional[int], Query()] = attr.ib(default=DEFAULT_LIMIT) + + +def patch_and_convert(interval: Optional[str]) -> Optional[DateTimeType]: + """Patch datetime to add hh-mm-ss and timezone info.""" + if interval: + interval = _patch_datetime(interval) + return str_to_interval(interval) @attr.s class PCSearchGetRequest(BaseSearchGetRequest): - limit: Optional[int] = attr.ib(default=DEFAULT_LIMIT) # type:ignore + datetime: Annotated[Optional[DateTimeType], Query()] = attr.ib( + default=None, converter=patch_and_convert + ) + limit: Annotated[Optional[int], Query()] = attr.ib(default=DEFAULT_LIMIT) diff --git a/pcstac/requirements-server.txt b/pcstac/requirements-server.txt index 3be02356..c662c22a 100644 --- a/pcstac/requirements-server.txt +++ b/pcstac/requirements-server.txt @@ -4,7 +4,9 @@ # # pip-compile --extra=server --output-file=pcstac/requirements-server.txt ./pcstac/setup.py # -anyio==3.7.1 +annotated-types==0.7.0 + # via pydantic +anyio==4.4.0 # via # starlette # watchfiles @@ -14,8 +16,6 @@ asyncpg==0.29.0 # via stac-fastapi-pgstac attrs==23.2.0 # via - # stac-fastapi-api - # stac-fastapi-extensions # stac-fastapi-pgstac # stac-fastapi-types brotli==1.1.0 @@ -34,13 +34,13 @@ click==8.1.7 # uvicorn dateparser==1.2.0 # via pygeofilter -exceptiongroup==1.2.0 +exceptiongroup==1.2.1 # via anyio -fastapi==0.110.0 +fastapi-slim==0.111.0 # via stac-fastapi-types fire==0.4.0 # via pypgstac -geojson-pydantic==0.6.3 +geojson-pydantic==1.1.0 # via stac-pydantic h11==0.14.0 # via uvicorn @@ -50,7 +50,7 @@ idna==3.7 # via # anyio # pcstac (pcstac/setup.py) -iso8601==1.1.0 +iso8601==2.1.0 # via stac-fastapi-types lark==0.12.0 # via pygeofilter @@ -67,28 +67,28 @@ psycopg-binary==3.1.18 # via psycopg psycopg-pool==3.1.9 # via pypgstac -pydantic[dotenv]==1.10.14 +pydantic==2.7.4 # via - # fastapi + # fastapi-slim # geojson-pydantic + # pydantic-settings # pypgstac - # stac-fastapi-api - # stac-fastapi-extensions # stac-fastapi-pgstac - # stac-fastapi-types # stac-pydantic +pydantic-core==2.18.4 + # via pydantic +pydantic-settings==2.3.3 + # via stac-fastapi-types pygeofilter==0.2.1 # via stac-fastapi-pgstac -pygeoif==1.4.0 +pygeoif==1.5.0 # via pygeofilter -pypgstac[psycopg]==0.7.10 +pypgstac[psycopg]==0.8.6 # via # pcstac (pcstac/setup.py) # stac-fastapi-pgstac pystac==1.10.1 - # via - # pcstac (pcstac/setup.py) - # stac-fastapi-types + # via pcstac (pcstac/setup.py) python-dateutil==2.8.2 # via # dateparser @@ -96,59 +96,60 @@ python-dateutil==2.8.2 # pystac python-dotenv==1.0.1 # via - # pydantic + # pydantic-settings # uvicorn pytz==2024.1 # via dateparser pyyaml==6.0.1 # via uvicorn -regex==2023.12.25 +regex==2024.5.15 # via dateparser six==1.16.0 # via # fire # python-dateutil -smart-open==6.4.0 +smart-open==7.0.4 # via pypgstac sniffio==1.3.1 # via anyio -stac-fastapi-api==2.4.8 +stac-fastapi-api==3.0.0a3 # via # pcstac (pcstac/setup.py) # stac-fastapi-extensions # stac-fastapi-pgstac -stac-fastapi-extensions==2.4.8 +stac-fastapi-extensions==3.0.0a3 # via # pcstac (pcstac/setup.py) # stac-fastapi-pgstac -stac-fastapi-pgstac==2.4.9 +stac-fastapi-pgstac==3.0.0a2 # via pcstac (pcstac/setup.py) -stac-fastapi-types==2.4.8 +stac-fastapi-types==3.0.0a3 # via # pcstac (pcstac/setup.py) # stac-fastapi-api # stac-fastapi-extensions # stac-fastapi-pgstac -stac-pydantic==2.0.3 +stac-pydantic==3.1.0 # via - # stac-fastapi-api - # stac-fastapi-extensions # stac-fastapi-pgstac # stac-fastapi-types -starlette==0.36.3 +starlette==0.37.2 # via # brotli-asgi - # fastapi + # fastapi-slim tenacity==8.1.0 # via pypgstac termcolor==2.4.0 # via fire -typing-extensions==4.10.0 +typing-extensions==4.12.2 # via - # fastapi + # anyio + # fastapi-slim + # pcstac (pcstac/setup.py) # psycopg # psycopg-pool # pydantic + # pydantic-core # pygeoif # starlette # uvicorn @@ -164,3 +165,5 @@ watchfiles==0.22.0 # via uvicorn websockets==12.0 # via uvicorn +wrapt==1.16.0 + # via smart-open diff --git a/pcstac/setup.py b/pcstac/setup.py index 11cd128b..9f99f8cd 100644 --- a/pcstac/setup.py +++ b/pcstac/setup.py @@ -5,14 +5,15 @@ # Runtime requirements. inst_reqs = [ "idna>=3.7.0", - "stac-fastapi.api==2.4.8", - "stac-fastapi.extensions==2.4.8", - "stac-fastapi.pgstac==2.4.9", - "stac-fastapi.types==2.4.8", + "stac-fastapi.api==3.0.0b2", + "stac-fastapi.extensions==3.0.0b2", + "stac-fastapi.pgstac==3.0.0a4", + "stac-fastapi.types==3.0.0b2", "orjson==3.10.4", # Required due to some imports related to pypgstac CLI usage in startup script - "pypgstac[psycopg]>=0.7.10,<0.8", + "pypgstac[psycopg]>=0.8.5,<0.9", "pystac==1.10.1", + "typing_extensions>=4.6.1", ] extra_reqs = { diff --git a/pcstac/tests/conftest.py b/pcstac/tests/conftest.py index b76f64a1..ba6e75ff 100644 --- a/pcstac/tests/conftest.py +++ b/pcstac/tests/conftest.py @@ -8,7 +8,7 @@ import pytest from fastapi import FastAPI from fastapi.responses import ORJSONResponse -from httpx import AsyncClient +from httpx import ASGITransport, AsyncClient from pypgstac.db import PgstacDB from pypgstac.migrate import Migrate from stac_fastapi.api.models import create_get_request_model, create_post_request_model @@ -102,7 +102,9 @@ async def app(api_client) -> AsyncGenerator[FastAPI, None]: @pytest.fixture(scope="session") async def app_client(app) -> AsyncGenerator[AsyncClient, None]: async with AsyncClient( - app=app, base_url="http://test/stac", headers={"X-Forwarded-For": "127.0.0.1"} + transport=ASGITransport(app=app), + base_url="http://test/stac", + headers={"X-Forwarded-For": "127.0.0.1"}, ) as c: yield c diff --git a/pcstac/tests/resources/test_item.py b/pcstac/tests/resources/test_item.py index d0e84df5..a740e983 100644 --- a/pcstac/tests/resources/test_item.py +++ b/pcstac/tests/resources/test_item.py @@ -1,5 +1,5 @@ import json -from datetime import datetime, timedelta +from datetime import timedelta from typing import Callable, Dict from urllib.parse import parse_qs, urlparse @@ -7,11 +7,16 @@ import pytest from geojson_pydantic.geometries import Polygon from stac_fastapi.pgstac.models.links import CollectionLinks -from stac_pydantic.shared import DATETIME_RFC339 +from stac_pydantic.shared import UtcDatetime from starlette.requests import Request +from pydantic import TypeAdapter from pcstac.config import get_settings +# Use a TypeAdapter to parse any datetime strings in a consistent manner +UtcDatetimeAdapter = TypeAdapter(UtcDatetime) +DATETIME_RFC339 = "%Y-%m-%dT%H:%M:%S.%fZ" + @pytest.mark.asyncio async def test_get_collection(app_client, load_test_data: Callable): @@ -116,7 +121,9 @@ async def test_item_search_temporal_query_post(app_client): assert items_resp.status_code == 200 first_item = items_resp.json()["features"][0] - item_date = datetime.strptime(first_item["properties"]["datetime"], DATETIME_RFC339) + item_date = UtcDatetimeAdapter.validate_strings( + first_item["properties"]["datetime"] + ) item_date = item_date + timedelta(seconds=1) params = { @@ -138,7 +145,9 @@ async def test_item_search_temporal_window_post(app_client): assert items_resp.status_code == 200 first_item = items_resp.json()["features"][0] - item_date = datetime.strptime(first_item["properties"]["datetime"], DATETIME_RFC339) + item_date = UtcDatetimeAdapter.validate_strings( + first_item["properties"]["datetime"] + ) item_date_before = item_date - timedelta(seconds=1) item_date_after = item_date + timedelta(seconds=1) @@ -153,6 +162,31 @@ async def test_item_search_temporal_window_post(app_client): assert resp_json["features"][0]["id"] == first_item["id"] +@pytest.mark.asyncio +async def test_item_search_temporal_window_post_date_only(app_client): + """Test POST search with spatio-temporal query (core)""" + items_resp = await app_client.get("/collections/naip/items") + assert items_resp.status_code == 200 + + first_item = items_resp.json()["features"][0] + item_date = UtcDatetimeAdapter.validate_strings( + first_item["properties"]["datetime"] + ) + item_date_before = item_date - timedelta(days=1) + item_date_after = item_date + timedelta(days=1) + + params = { + "collections": [first_item["collection"]], + "intersects": first_item["geometry"], + "datetime": f"{item_date_before.strftime('%Y-%m-%d')}/" + f"{item_date_after.strftime('%Y-%m-%d')}", + } + resp = await app_client.post("/search", json=params) + assert resp.status_code == 200 + resp_json = resp.json() + assert resp_json["features"][0]["id"] == first_item["id"] + + @pytest.mark.asyncio async def test_item_search_by_id_get(app_client): """Test GET search by item id (core)""" @@ -212,7 +246,9 @@ async def test_item_search_temporal_window_get(app_client): assert items_resp.status_code == 200 first_item = items_resp.json()["features"][0] - item_date = datetime.strptime(first_item["properties"]["datetime"], DATETIME_RFC339) + item_date = UtcDatetimeAdapter.validate_strings( + first_item["properties"]["datetime"] + ) item_date_before = item_date - timedelta(seconds=1) item_date_after = item_date + timedelta(seconds=1) @@ -234,7 +270,9 @@ async def test_item_search_temporal_window_get_date_only(app_client): assert items_resp.status_code == 200 first_item = items_resp.json()["features"][0] - item_date = datetime.strptime(first_item["properties"]["datetime"], DATETIME_RFC339) + item_date = UtcDatetimeAdapter.validate_strings( + first_item["properties"]["datetime"] + ) item_date_before = item_date - timedelta(days=1) item_date_after = item_date + timedelta(days=1) @@ -247,10 +285,6 @@ async def test_item_search_temporal_window_get_date_only(app_client): resp = await app_client.get("/search", params=params) assert resp.status_code == 200 resp_json = resp.json() - import json - - print(json.dumps(resp_json, indent=2)) - assert resp_json["features"][0]["id"] == first_item["id"] @@ -454,8 +488,11 @@ async def test_pagination_token_idempotent(app_client): # Construct a search that should return all items, but limit to a few # so that a "next" link is returned page = await app_client.get( - "/search", params={"datetime": "1900-01-01/2030-01-01", "limit": 3} + "/search", + params={"datetime": "1900-01-01T00:00:00Z/2030-01-01T00:00:00Z", "limit": 3}, ) + assert page.status_code == 200 + # Get the next link page_data = page.json() next_link = list(filter(lambda l: l["rel"] == "next", page_data["links"])) @@ -500,7 +537,7 @@ async def test_field_extension_exclude_default_includes(app_client): async def test_search_intersects_and_bbox(app_client): """Test POST search intersects and bbox are mutually exclusive (core)""" bbox = [-118, 34, -117, 35] - geoj = Polygon.from_bounds(*bbox).dict(exclude_none=True) + geoj = Polygon.from_bounds(*bbox).model_dump(exclude_none=True) params = {"bbox": bbox, "intersects": geoj} resp = await app_client.post("/search", json=params) assert resp.status_code == 400 diff --git a/pcstac/tests/test_rate_limit.py b/pcstac/tests/test_rate_limit.py index 8dc41031..bc5977df 100644 --- a/pcstac/tests/test_rate_limit.py +++ b/pcstac/tests/test_rate_limit.py @@ -2,7 +2,7 @@ import pytest from fastapi import FastAPI -from httpx import AsyncClient +from httpx import ASGITransport, AsyncClient from pccommon.constants import HTTP_429_TOO_MANY_REQUESTS @@ -13,7 +13,9 @@ async def test_rate_limit_collection(app: FastAPI): # set the ip to one that doesn't have the rate limit exception async with AsyncClient( - app=app, base_url="http://test", headers={"X-Forwarded-For": "127.0.0.2"} + transport=ASGITransport(app=app), + base_url="http://test", + headers={"X-Forwarded-For": "127.0.0.2"}, ) as app_client: resp = None for _ in range(0, 400): @@ -40,7 +42,9 @@ async def test_rate_limit_collection_ip_Exception(app_client: AsyncClient): async def test_reregistering_rate_limit_script(app: FastAPI, app_client: AsyncClient): # set the ip to one that doesn't have the rate limit exception async with AsyncClient( - app=app, base_url="http://test", headers={"X-Forwarded-For": "127.0.0.2"} + transport=ASGITransport(app=app), + base_url="http://test", + headers={"X-Forwarded-For": "127.0.0.2"}, ) as app_client: async def _hash_exists(): diff --git a/pctiler/pctiler/colormaps/__init__.py b/pctiler/pctiler/colormaps/__init__.py index 58436a70..4cae268d 100644 --- a/pctiler/pctiler/colormaps/__init__.py +++ b/pctiler/pctiler/colormaps/__init__.py @@ -1,10 +1,8 @@ -from enum import Enum -from typing import Dict, Optional +from typing import Dict -from fastapi import Query from rio_tiler.colormap import cmap from rio_tiler.types import ColorMapType -from titiler.core.dependencies import ColorMapParams +from titiler.core.dependencies import create_colormap_dependency from .alos_palsar_mosaic import alos_palsar_mosaic_colormaps from .chloris import chloris_colormaps @@ -41,21 +39,8 @@ for k, v in custom_colormaps.items(): registered_cmaps = registered_cmaps.register({k: v}) -PCColorMapNames = Enum( # type: ignore - "ColorMapNames", [(a, a) for a in sorted(registered_cmaps.list())] -) - - -def PCColorMapParams( - colormap_name: PCColorMapNames = Query(None, description="Colormap name"), - colormap: str = Query(None, description="JSON encoded custom Colormap"), -) -> Optional[ColorMapType]: - if colormap_name: - cm = custom_colormaps.get(colormap_name.value) - if cm: - return cm - return ColorMapParams(colormap_name, colormap) +PCColorMapParams = create_colormap_dependency(registered_cmaps) # Placeholder for non-discrete range colormaps (unsupported) # "hgb-above": { diff --git a/pctiler/pctiler/config.py b/pctiler/pctiler/config.py index 87a89932..9eef28a7 100644 --- a/pctiler/pctiler/config.py +++ b/pctiler/pctiler/config.py @@ -4,7 +4,8 @@ from urllib.parse import urljoin from fastapi import Request -from pydantic import BaseSettings, Field +from pydantic import Field +from pydantic_settings import BaseSettings # Hostname to fetch STAC information from STAC_API_URL_ENV_VAR = "STAC_API_URL" @@ -39,14 +40,21 @@ class Settings(BaseSettings): mosaic_endpoint_prefix: str = "/mosaic" legend_endpoint_prefix: str = "/legend" vector_tile_endpoint_prefix: str = "/vector" - vector_tile_sa_base_url: str = Field(env=VECTORTILE_SA_BASE_URL_ENV_VAR, default="") + vector_tile_sa_base_url: str = Field( + default="", + json_schema_extra={"env": VECTORTILE_SA_BASE_URL_ENV_VAR}, + ) debug: bool = os.getenv("TILER_DEBUG", "False").lower() == "true" api_version: str = "1.0" default_max_items_per_tile: int = Field( - env=DEFAULT_MAX_ITEMS_PER_TILE_ENV_VAR, default=10 + default=10, + json_schema_extra={"env": DEFAULT_MAX_ITEMS_PER_TILE_ENV_VAR}, + ) + request_timeout: int = Field( + default=30, + json_schema_extra={"env": REQUEST_TIMEOUT_ENV_VAR}, ) - request_timeout: int = Field(env=REQUEST_TIMEOUT_ENV_VAR, default=30) feature_flags: FeatureFlags = FeatureFlags() diff --git a/pctiler/pctiler/endpoints/item.py b/pctiler/pctiler/endpoints/item.py index e9d8ef9e..5e640372 100644 --- a/pctiler/pctiler/endpoints/item.py +++ b/pctiler/pctiler/endpoints/item.py @@ -1,11 +1,12 @@ from urllib.parse import quote_plus, urljoin +import pystac from fastapi import Query, Request, Response from fastapi.templating import Jinja2Templates from html_sanitizer.sanitizer import Sanitizer from starlette.responses import HTMLResponse from titiler.core.factory import MultiBaseTilerFactory -from titiler.pgstac.dependencies import ItemPathParams # removed in titiler.pgstac 3.0 +from titiler.pgstac.dependencies import get_stac_item from pccommon.config import get_render_config from pctiler.colormaps import PCColorMapParams @@ -19,6 +20,15 @@ from importlib_resources import files as resources_files # type: ignore +def ItemPathParams( + request: Request, + collection: str = Query(..., description="STAC Collection ID"), + item: str = Query(..., description="STAC Item ID"), +) -> pystac.Item: + """STAC Item dependency.""" + return get_stac_item(request.app.state.dbpool, collection, item) + + # TODO: mypy fails in python 3.9, we need to find a proper way to do this templates = Jinja2Templates( directory=str(resources_files(__package__) / "templates") # type: ignore @@ -65,9 +75,9 @@ def map( ) return templates.TemplateResponse( - "item_preview.html", + request, + name="item_preview.html", context={ - "request": request, "tileJson": tilejson_url, "collectionId": collection_sanitized, "itemId": item_sanitized, diff --git a/pctiler/pctiler/endpoints/legend.py b/pctiler/pctiler/endpoints/legend.py index 30cbb81b..fed317e4 100644 --- a/pctiler/pctiler/endpoints/legend.py +++ b/pctiler/pctiler/endpoints/legend.py @@ -1,3 +1,5 @@ +# NOTE: we now have https://developmentseed.org/titiler/endpoints/colormaps/ in titiler + from io import BytesIO from typing import Sequence diff --git a/pctiler/pctiler/endpoints/pg_mosaic.py b/pctiler/pctiler/endpoints/pg_mosaic.py index fd49a42e..e8fb57b6 100644 --- a/pctiler/pctiler/endpoints/pg_mosaic.py +++ b/pctiler/pctiler/endpoints/pg_mosaic.py @@ -1,6 +1,7 @@ +from typing import Optional, List from dataclasses import dataclass, field -from fastapi import Query, Request +from fastapi import FastAPI, Path, Query, Request from fastapi.responses import ORJSONResponse from psycopg_pool import ConnectionPool from titiler.core import dependencies @@ -19,6 +20,11 @@ class AssetsBidxExprParams(dependencies.AssetsBidxExprParams): collection: str = Query(None, description="STAC Collection ID") +def PathParams(searchid: str = Path(..., description="Search Id")) -> str: + """SearchId""" + return searchid + + @dataclass(init=False) class BackendParams(dependencies.DefaultDependency): """backend parameters.""" @@ -34,31 +40,42 @@ def __init__(self, request: Request): pgstac_mosaic_factory = MosaicTilerFactory( reader=PGSTACBackend, + path_dependency=PathParams, colormap_dependency=PCColorMapParams, layer_dependency=AssetsBidxExprParams, reader_dependency=ReaderParams, - router_prefix=get_settings().mosaic_endpoint_prefix, + router_prefix=get_settings().mosaic_endpoint_prefix + "/{searchid}", backend_dependency=BackendParams, - add_map_viewer=False, add_statistics=False, - add_mosaic_list=False, ) -@pgstac_mosaic_factory.router.get( - "/info", response_model=MosaicInfo, response_class=ORJSONResponse -) -def mosaic_info( - request: Request, collection: str = Query(..., description="STAC Collection ID") -) -> ORJSONResponse: - collection_config = get_collection_config(collection) - if not collection_config or not collection_config.mosaic_info: - return ORJSONResponse( - status_code=404, - content=f"No mosaic info available for collection {collection}", - ) +def add_collection_mosaic_info_route( + app: FastAPI, + *, + prefix: str = "", + tags: Optional[List[str]] = None, +) -> None: + """add `/info` endpoint.""" - return ORJSONResponse( - status_code=200, - content=collection_config.mosaic_info.dict(by_alias=True, exclude_unset=True), + @app.get( + f"{prefix}/info", + response_model=MosaicInfo, + response_class=ORJSONResponse, ) + def mosaic_info( + request: Request, collection: str = Query(..., description="STAC Collection ID") + ) -> ORJSONResponse: + collection_config = get_collection_config(collection) + if not collection_config or not collection_config.mosaic_info: + return ORJSONResponse( + status_code=404, + content=f"No mosaic info available for collection {collection}", + ) + + return ORJSONResponse( + status_code=200, + content=collection_config.mosaic_info.model_dump( + by_alias=True, exclude_unset=True + ), + ) diff --git a/pctiler/pctiler/endpoints/vector_tiles.py b/pctiler/pctiler/endpoints/vector_tiles.py index 0a82db26..9f2e8d2b 100644 --- a/pctiler/pctiler/endpoints/vector_tiles.py +++ b/pctiler/pctiler/endpoints/vector_tiles.py @@ -60,7 +60,7 @@ async def get_tilejson( if tileset.center: tilejson["center"] = tileset.center - return tilejson + return TileJSON(**tilejson) @vector_tile_router.get( diff --git a/pctiler/pctiler/main.py b/pctiler/pctiler/main.py index 93d1da3b..ea36ce11 100755 --- a/pctiler/pctiler/main.py +++ b/pctiler/pctiler/main.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 import logging import os -from typing import Dict, List +from contextlib import asynccontextmanager +from typing import Dict, List, AsyncGenerator from fastapi import FastAPI from fastapi.openapi.utils import get_openapi @@ -40,18 +41,24 @@ settings = get_settings() + +@asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncGenerator: + """FastAPI Lifespan.""" + await connect_to_db(app) + yield + await close_db_connection(app) + + app = FastAPI( title=settings.title, openapi_url=settings.openapi_url, root_path=APP_ROOT_PATH, + lifespan=lifespan, ) app.state.service_name = ServiceName.TILER -# Note: -# With titiler.pgstac >3.0, items endpoint has changed and use path-parameter -# /collections/{collectionId}/items/{itemId} instead of query-parameter -# https://github.com/stac-utils/titiler-pgstac/blob/d16102bf331ba588f31e131e65b07637d649b4bd/titiler/pgstac/main.py#L87-L92 app.include_router( item.pc_tile_factory.router, prefix=settings.item_endpoint_prefix, @@ -63,6 +70,11 @@ prefix=settings.mosaic_endpoint_prefix, tags=["PgSTAC Mosaic endpoints"], ) +pg_mosaic.add_collection_mosaic_info_route( + app, + prefix=settings.mosaic_endpoint_prefix, + tags=["PgSTAC Mosaic endpoints"], +) app.include_router( legend.legend_router, @@ -109,18 +121,6 @@ ) -@app.on_event("startup") -async def startup_event() -> None: - """Connect to database on startup.""" - await connect_to_db(app) - - -@app.on_event("shutdown") -async def shutdown_event() -> None: - """Close database connection.""" - await close_db_connection(app) - - @app.get("/") async def read_root() -> Dict[str, str]: return {"Hello": "Planetary Developer!"} diff --git a/pctiler/pctiler/reader.py b/pctiler/pctiler/reader.py index 19724a8c..c1443541 100644 --- a/pctiler/pctiler/reader.py +++ b/pctiler/pctiler/reader.py @@ -49,22 +49,15 @@ class ItemSTACReader(PgSTACReader): def _get_asset_info(self, asset: str) -> AssetInfo: """return asset's url.""" - asset_url = BlobCDN.transform_if_available( - super()._get_asset_info(asset)["url"] - ) + info = super()._get_asset_info(asset) + asset_url = BlobCDN.transform_if_available(info["url"]) if self.input.collection_id: render_config = get_render_config(self.input.collection_id) if render_config and render_config.requires_token: asset_url = pc.sign(asset_url) - asset_info = self.input.assets[asset] - info = AssetInfo(url=asset_url) - - if "file:header_size" in asset_info.extra_fields: - h = asset_info.extra_fields["file:header_size"] - info["env"] = {"GDAL_INGESTED_BYTES_AT_OPEN": h} - + info["url"] = asset_url return info @@ -119,7 +112,7 @@ class PGSTACBackend(pgstac_mosaic.PGSTACBackend): request: Optional[Request] = attr.ib(default=None) # Override from PGSTACBackend to use collection - def assets_for_tile( + def assets_for_tile( # type: ignore self, x: int, y: int, z: int, collection: Optional[str] = None, **kwargs: Any ) -> List[Dict]: settings = get_settings() @@ -166,12 +159,11 @@ def assets_for_tile( return assets # override from PGSTACBackend to pass through collection - def tile( + def tile( # type: ignore self, tile_x: int, tile_y: int, tile_z: int, - reverse: bool = False, collection: Optional[str] = None, scan_limit: Optional[int] = None, items_limit: Optional[int] = None, @@ -199,8 +191,6 @@ def tile( ) ts = time.perf_counter() - if reverse: - mosaic_assets = list(reversed(mosaic_assets)) def _reader( item: Dict[str, Any], x: int, y: int, z: int, **kwargs: Any diff --git a/pctiler/requirements-dev.txt b/pctiler/requirements-dev.txt index 511c8b7a..a3394c3d 100644 --- a/pctiler/requirements-dev.txt +++ b/pctiler/requirements-dev.txt @@ -5,9 +5,9 @@ # pip-compile --extra=dev --output-file=pctiler/requirements-dev.txt ./pctiler/setup.py # affine==2.4.0 - # via - # rasterio - # supermercado + # via rasterio +annotated-types==0.7.0 + # via pydantic anyio==4.3.0 # via # httpx @@ -18,11 +18,11 @@ attrs==23.2.0 # morecantile # rasterio # rio-tiler -boto3==1.34.123 +boto3==1.34.136 # via # pctiler (pctiler/setup.py) # rio-tiler -botocore==1.34.123 +botocore==1.34.136 # via # boto3 # pctiler (pctiler/setup.py) @@ -30,7 +30,6 @@ botocore==1.34.123 cachetools==5.3.3 # via # cogeo-mosaic - # morecantile # rio-tiler certifi==2024.2.2 # via @@ -45,20 +44,13 @@ click==8.1.7 # via # click-plugins # cligj - # mercantile # planetary-computer # rasterio - # stac-pydantic - # supermercado click-plugins==1.1.1 - # via - # rasterio - # supermercado + # via rasterio cligj==0.7.2 - # via - # rasterio - # supermercado -cogeo-mosaic==5.0.0 + # via rasterio +cogeo-mosaic==7.1.0 # via titiler-mosaic color-operations==0.1.3 # via rio-tiler @@ -68,16 +60,15 @@ cycler==0.12.1 # via matplotlib exceptiongroup==1.2.0 # via anyio -fastapi==0.91.0 +fastapi-slim==0.111.0 # via + # pctiler (pctiler/setup.py) # titiler-core - # titiler-pgstac fonttools==4.53.0 # via matplotlib -geojson-pydantic==0.4.2 +geojson-pydantic==1.1.0 # via # pctiler (pctiler/setup.py) - # stac-pydantic # titiler-core # titiler-pgstac h11==0.14.0 @@ -112,12 +103,12 @@ markupsafe==2.1.5 # via jinja2 matplotlib==3.9.0 # via pctiler (pctiler/setup.py) -mercantile==1.2.1 - # via supermercado -morecantile==3.4.0 +morecantile==5.3.0 # via # cogeo-mosaic # rio-tiler + # supermorecado + # titiler-core numexpr==2.9.0 # via rio-tiler numpy==1.26.4 @@ -130,17 +121,18 @@ numpy==1.26.4 # rio-tiler # shapely # snuggs - # supermercado # titiler-core orjson==3.10.4 # via pctiler (pctiler/setup.py) packaging==24.1 - # via matplotlib + # via + # matplotlib + # planetary-computer pillow==10.3.0 # via # matplotlib # pctiler (pctiler/setup.py) -planetary-computer==0.4.9 +planetary-computer==1.0.0 # via pctiler (pctiler/setup.py) psycopg[binary,pool]==3.1.18 # via pctiler (pctiler/setup.py) @@ -148,17 +140,24 @@ psycopg-binary==3.1.18 # via psycopg psycopg-pool==3.2.1 # via psycopg -pydantic[dotenv]==1.10.14 +pydantic==2.7.4 # via # cogeo-mosaic - # fastapi + # fastapi-slim # geojson-pydantic # morecantile # pctiler (pctiler/setup.py) # planetary-computer + # pydantic-settings # rio-tiler - # stac-pydantic # titiler-core + # titiler-pgstac +pydantic-core==2.18.4 + # via pydantic +pydantic-settings==2.3.3 + # via + # cogeo-mosaic + # titiler-pgstac pyparsing==3.1.2 # via # matplotlib @@ -180,7 +179,9 @@ python-dateutil==2.9.0.post0 # pystac # pystac-client python-dotenv==1.0.1 - # via pydantic + # via + # planetary-computer + # pydantic-settings pytz==2024.1 # via planetary-computer rasterio==1.3.10 @@ -188,14 +189,14 @@ rasterio==1.3.10 # cogeo-mosaic # pctiler (pctiler/setup.py) # rio-tiler - # supermercado + # supermorecado # titiler-core -requests==2.32.2 +requests==2.32.3 # via # pctiler (pctiler/setup.py) # planetary-computer # pystac-client -rio-tiler==4.1.13 +rio-tiler==6.6.1 # via # cogeo-mosaic # titiler-core @@ -213,24 +214,20 @@ sniffio==1.3.1 # httpx snuggs==1.4.7 # via rasterio -stac-pydantic==2.0.3 - # via titiler-pgstac -starlette==0.24.0 - # via - # fastapi - # titiler-pgstac -supermercado==0.2.0 +starlette==0.37.2 + # via fastapi-slim +supermorecado==0.1.2 # via cogeo-mosaic -titiler-core==0.10.2 +titiler-core==0.18.3 # via # pctiler (pctiler/setup.py) # titiler-mosaic # titiler-pgstac -titiler-mosaic==0.10.2 +titiler-mosaic==0.18.3 # via # pctiler (pctiler/setup.py) # titiler-pgstac -titiler-pgstac==0.2.4 +titiler-pgstac==1.3.0 # via pctiler (pctiler/setup.py) types-requests==2.31.0.6 # via pctiler (pctiler/setup.py) @@ -239,10 +236,13 @@ types-urllib3==1.26.25.14 typing-extensions==4.10.0 # via # anyio + # fastapi-slim # psycopg # psycopg-pool # pydantic + # pydantic-core # starlette + # titiler-core urllib3==1.26.18 # via # botocore diff --git a/pctiler/requirements-server.txt b/pctiler/requirements-server.txt index 0ab6f9dd..4051a9bc 100644 --- a/pctiler/requirements-server.txt +++ b/pctiler/requirements-server.txt @@ -5,9 +5,9 @@ # pip-compile --extra=server --output-file=pctiler/requirements-server.txt ./pctiler/setup.py # affine==2.4.0 - # via - # rasterio - # supermercado + # via rasterio +annotated-types==0.7.0 + # via pydantic anyio==3.7.1 # via # httpx @@ -21,11 +21,11 @@ attrs==23.2.0 # morecantile # rasterio # rio-tiler -boto3==1.34.123 +boto3==1.34.136 # via # pctiler (pctiler/setup.py) # rio-tiler -botocore==1.34.123 +botocore==1.34.136 # via # boto3 # pctiler (pctiler/setup.py) @@ -33,7 +33,6 @@ botocore==1.34.123 cachetools==5.3.3 # via # cogeo-mosaic - # morecantile # rio-tiler certifi==2024.2.2 # via @@ -48,21 +47,14 @@ click==8.1.7 # via # click-plugins # cligj - # mercantile # planetary-computer # rasterio - # stac-pydantic - # supermercado # uvicorn click-plugins==1.1.1 - # via - # rasterio - # supermercado + # via rasterio cligj==0.7.2 - # via - # rasterio - # supermercado -cogeo-mosaic==5.0.0 + # via rasterio +cogeo-mosaic==7.1.0 # via titiler-mosaic color-operations==0.1.3 # via rio-tiler @@ -72,16 +64,15 @@ cycler==0.12.1 # via matplotlib exceptiongroup==1.2.0 # via anyio -fastapi==0.91.0 +fastapi-slim==0.111.0 # via + # pctiler (pctiler/setup.py) # titiler-core - # titiler-pgstac fonttools==4.53.0 # via matplotlib -geojson-pydantic==0.4.2 +geojson-pydantic==1.1.0 # via # pctiler (pctiler/setup.py) - # stac-pydantic # titiler-core # titiler-pgstac h11==0.14.0 @@ -120,12 +111,12 @@ markupsafe==2.1.5 # via jinja2 matplotlib==3.9.0 # via pctiler (pctiler/setup.py) -mercantile==1.2.1 - # via supermercado -morecantile==3.4.0 +morecantile==5.3.0 # via # cogeo-mosaic # rio-tiler + # supermorecado + # titiler-core numexpr==2.9.0 # via rio-tiler numpy==1.26.4 @@ -138,17 +129,18 @@ numpy==1.26.4 # rio-tiler # shapely # snuggs - # supermercado # titiler-core orjson==3.10.4 # via pctiler (pctiler/setup.py) packaging==24.1 - # via matplotlib + # via + # matplotlib + # planetary-computer pillow==10.3.0 # via # matplotlib # pctiler (pctiler/setup.py) -planetary-computer==0.4.9 +planetary-computer==1.0.0 # via pctiler (pctiler/setup.py) psycopg[binary,pool]==3.1.18 # via pctiler (pctiler/setup.py) @@ -156,17 +148,24 @@ psycopg-binary==3.1.18 # via psycopg psycopg-pool==3.2.1 # via psycopg -pydantic[dotenv]==1.10.14 +pydantic==2.7.4 # via # cogeo-mosaic - # fastapi + # fastapi-slim # geojson-pydantic # morecantile # pctiler (pctiler/setup.py) # planetary-computer + # pydantic-settings # rio-tiler - # stac-pydantic # titiler-core + # titiler-pgstac +pydantic-core==2.18.4 + # via pydantic +pydantic-settings==2.3.3 + # via + # cogeo-mosaic + # titiler-pgstac pyparsing==3.1.2 # via # matplotlib @@ -189,7 +188,8 @@ python-dateutil==2.9.0.post0 # pystac-client python-dotenv==1.0.1 # via - # pydantic + # planetary-computer + # pydantic-settings # uvicorn pytz==2024.1 # via planetary-computer @@ -200,14 +200,14 @@ rasterio==1.3.10 # cogeo-mosaic # pctiler (pctiler/setup.py) # rio-tiler - # supermercado + # supermorecado # titiler-core -requests==2.32.2 +requests==2.32.3 # via # pctiler (pctiler/setup.py) # planetary-computer # pystac-client -rio-tiler==4.1.13 +rio-tiler==6.6.1 # via # cogeo-mosaic # titiler-core @@ -225,33 +225,31 @@ sniffio==1.3.1 # httpx snuggs==1.4.7 # via rasterio -stac-pydantic==2.0.3 - # via titiler-pgstac -starlette==0.24.0 - # via - # fastapi - # titiler-pgstac -supermercado==0.2.0 +starlette==0.37.2 + # via fastapi-slim +supermorecado==0.1.2 # via cogeo-mosaic -titiler-core==0.10.2 +titiler-core==0.18.3 # via # pctiler (pctiler/setup.py) # titiler-mosaic # titiler-pgstac -titiler-mosaic==0.10.2 +titiler-mosaic==0.18.3 # via # pctiler (pctiler/setup.py) # titiler-pgstac -titiler-pgstac==0.2.4 +titiler-pgstac==1.3.0 # via pctiler (pctiler/setup.py) typing-extensions==4.10.0 # via # asgiref + # fastapi-slim # psycopg # psycopg-pool # pydantic + # pydantic-core # starlette -urllib3==1.26.18 +urllib3==1.26.19 # via # botocore # requests diff --git a/pctiler/setup.py b/pctiler/setup.py index 89c93ad8..ae684772 100644 --- a/pctiler/setup.py +++ b/pctiler/setup.py @@ -5,26 +5,25 @@ # Runtime requirements, see environment.yaml inst_reqs: List[str] = [ - "geojson-pydantic==0.4.2", + "fastapi-slim==0.111.0", + "geojson-pydantic==1.1.0", "jinja2==3.1.4", "pystac==1.10.1", - "planetary-computer==0.4.9", + "planetary-computer==1.0.0", "rasterio==1.3.10", - "titiler.core==0.10.2", - "titiler.mosaic==0.10.2", + "titiler.core==0.18.3", + "titiler.mosaic==0.18.3", "pillow==10.3.0", - "boto3==1.34.123", - "botocore==1.34.123", - "pydantic==1.10.14", + "boto3==1.34.136", + "botocore==1.34.136", + "pydantic>=2.7,<2.8", "idna>=3.7.0", - "requests==2.32.2", + "requests==2.32.3", # titiler-pgstac "psycopg[binary,pool]", - "titiler.pgstac==0.2.4", - + "titiler.pgstac==1.3.0", # colormap dependencies "matplotlib==3.9.0", - "orjson==3.10.4", "importlib_resources>=1.1.0;python_version<'3.9'", ] diff --git a/pctiler/tests/conftest.py b/pctiler/tests/conftest.py index 95d50231..4c75dbe7 100644 --- a/pctiler/tests/conftest.py +++ b/pctiler/tests/conftest.py @@ -1,7 +1,7 @@ from typing import List import pytest -from httpx import AsyncClient +from httpx import ASGITransport, AsyncClient from pytest import Config, Item, Parser @@ -36,6 +36,9 @@ async def client() -> AsyncClient: from pctiler.main import app await connect_to_db(app) - async with AsyncClient(app=app, base_url="http://test") as client: + async with AsyncClient( + transport=ASGITransport(app=app), + base_url="http://test", + ) as client: yield client await close_db_connection(app) diff --git a/requirements-dev.txt b/requirements-dev.txt index fd0c6386..d2af00a2 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ black==24.4.2 flake8==3.8.4 isort==5.9.2 -mypy==1.8.0 +mypy==1.10.0 openapi-spec-validator==0.3.0 pytest==7.* pytest-asyncio==0.18.* diff --git a/scripts/bin/test-funcs b/scripts/bin/test-funcs index 22b1d2db..64bb06eb 100755 --- a/scripts/bin/test-funcs +++ b/scripts/bin/test-funcs @@ -15,6 +15,23 @@ This scripts is meant to be run inside the funcs container. " } +while [[ $# -gt 0 ]]; do case $1 in + --no-integration) + INTEGRATION="--no-integration" + shift + ;; + --help) + usage + exit 0 + shift + ;; + *) + usage "Unknown parameter passed: $1" + shift + shift + ;; + esac done + if [ "${BASH_SOURCE[0]}" = "${0}" ]; then echo "Running mypy for funcs..." @@ -27,6 +44,6 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then flake8 pcfuncs echo "Running unit tests for funcs..." - python -m pytest pcfuncs/tests + python -m pytest pcfuncs/tests ${INTEGRATION:+$INTEGRATION} fi diff --git a/scripts/cideploy b/scripts/cideploy index 053695fc..3954187f 100755 --- a/scripts/cideploy +++ b/scripts/cideploy @@ -73,6 +73,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then # Run deployment script ${DOCKER_COMPOSE} run --rm \ deploy bin/deploy \ - -t "${TERRAFORM_DIR}" + -t "${TERRAFORM_DIR}" \ + -y ) fi diff --git a/scripts/test b/scripts/test index 35685dff..8f283525 100755 --- a/scripts/test +++ b/scripts/test @@ -94,7 +94,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then docker-compose \ -f docker-compose.yml \ run --rm \ - funcs /bin/bash -c "cd /opt/src && scripts/bin/test-funcs" + funcs /bin/bash -c "cd /opt/src && scripts/bin/test-funcs ${NO_INTEGRATION}" fi fi