From 047363e2325f78629c2ecbb02575bdf6cc79009d Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 3 Jul 2024 14:47:47 -0500 Subject: [PATCH 1/7] style: flake8 updates --- .../pipelines/aws_noaa_oisst_avhrr_only/test_app.py | 2 +- .../pipelines/aws_noaa_oisst_avhrr_only/test_collection.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_app.py b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_app.py index fcbd340..a83a952 100644 --- a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_app.py +++ b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_app.py @@ -12,7 +12,7 @@ def test_create_item(): item = module_create_item() - assert type(item) == pystac.item.Item + assert type(item) is pystac.item.Item @pytest.mark.parametrize("pipeline_id", ["aws_noaa_oisst_avhrr_only"]) diff --git a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_collection.py b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_collection.py index c4e1584..d544e70 100644 --- a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_collection.py +++ b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_collection.py @@ -12,7 +12,7 @@ def test_create_collection(): collection = module_create_collection() - assert type(collection) == pystac.collection.Collection + assert type(collection) is pystac.collection.Collection @pytest.mark.parametrize("pipeline_id", ["aws_noaa_oisst_avhrr_only"]) From 50b17b09f2786c3e6980e22519f4f6c6d25bee51 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 3 Jul 2024 14:48:32 -0500 Subject: [PATCH 2/7] chore: update amazonia_1 pipeline --- stactools_pipelines/pipelines/amazonia_1/config.yaml | 2 +- stactools_pipelines/pipelines/amazonia_1/requirements.txt | 2 +- stactools_pipelines/pipelines/amazonia_1/test_app.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/stactools_pipelines/pipelines/amazonia_1/config.yaml b/stactools_pipelines/pipelines/amazonia_1/config.yaml index 59fde3a..826c654 100644 --- a/stactools_pipelines/pipelines/amazonia_1/config.yaml +++ b/stactools_pipelines/pipelines/amazonia_1/config.yaml @@ -3,7 +3,7 @@ compute: "awslambda" secret_arn: "arn:aws:secretsmanager:eu-central-1:751953039297:secret:asdi-auth-stack-alukach/asdi-workflows-6awSos" ingestor_url: "https://crtx3178aa.execute-api.us-west-2.amazonaws.com/dev" - sns: "arn:aws:sns:us-east-1:599544552497:NewAMQuicklook" + sns: "arn:aws:sns:us-west-2:599544552497:NewAM1Quicklook" # inventory_location: "" # historic_frequency: 1 # initial_chunk: "" diff --git a/stactools_pipelines/pipelines/amazonia_1/requirements.txt b/stactools_pipelines/pipelines/amazonia_1/requirements.txt index 123ceae..dc524c0 100644 --- a/stactools_pipelines/pipelines/amazonia_1/requirements.txt +++ b/stactools_pipelines/pipelines/amazonia_1/requirements.txt @@ -1,4 +1,4 @@ aws-lambda-powertools jmespath -stactools-amazonia-1 >= 0.1.2 +stactools-amazonia-1 >= 0.1.3 s3fs diff --git a/stactools_pipelines/pipelines/amazonia_1/test_app.py b/stactools_pipelines/pipelines/amazonia_1/test_app.py index 4f587cc..6071fac 100644 --- a/stactools_pipelines/pipelines/amazonia_1/test_app.py +++ b/stactools_pipelines/pipelines/amazonia_1/test_app.py @@ -33,7 +33,7 @@ def test_xml_key_from_quicklook_key(): @pytest.mark.parametrize("pipeline_id", ["amazonia_1"]) @pytest.mark.parametrize("module", ["app"]) -@pytest.mark.parametrize("bucket", ["amazonia-pds"]) +@pytest.mark.parametrize("bucket", ["brazil-eosats"]) @pytest.mark.parametrize("key", [KEY]) def test_handler( mock_env, # pylint: disable=unused-argument @@ -46,7 +46,7 @@ def test_handler( handler(event=sqs_event, context={}) get_token.assert_called_once() create_item.assert_called_once_with( - asset_href="s3://amazonia-pds/AMAZONIA1/WFI/035/020/" + asset_href="s3://brazil-eosats/AMAZONIA1/WFI/035/020/" "AMAZONIA_1_WFI_20220814_035_020_L4/AMAZONIA_1_WFI_20220814_035_020_L4_BAND2.xml" ) requests.post.assert_called_once_with( From 3dadfa1af27f9d5b8ffb4361ac16b380b29fd1d9 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 3 Jul 2024 14:49:08 -0500 Subject: [PATCH 3/7] chore: update to model_validators --- stactools_pipelines/models/pipeline.py | 29 +++++++++++++------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/stactools_pipelines/models/pipeline.py b/stactools_pipelines/models/pipeline.py index 4359331..eb3a3b8 100644 --- a/stactools_pipelines/models/pipeline.py +++ b/stactools_pipelines/models/pipeline.py @@ -1,7 +1,8 @@ from enum import Enum from typing import Optional +from typing_extensions import Self -from pydantic import BaseModel, root_validator +from pydantic import BaseModel, model_validator class ComputeEnum(str, Enum): @@ -13,22 +14,22 @@ class Pipeline(BaseModel): compute: ComputeEnum secret_arn: str ingestor_url: str - sns: Optional[str] - inventory_location: Optional[str] - historic_frequency: Optional[int] - initial_chunk: Optional[str] + sns: Optional[str] = None + inventory_location: Optional[str] = None + historic_frequency: Optional[int] = None + initial_chunk: Optional[str] = None - @root_validator(pre=False) - def historic_frequency_if_inventory_location(cls, values): - if values["inventory_location"] and values["historic_frequency"] is None: + @model_validator(mode="after") + def historic_frequency_if_inventory_location(self) -> Self: + if self.inventory_location and self.historic_frequency is None: raise ValueError("Must include historic_frequency with inventory_location") - return values + return self - @root_validator(pre=False) - def initial_chunk_if_historic_frequency_greater_than_0(cls, values): - if values["historic_frequency"]: - if values["historic_frequency"] > 0 and values["initial_chunk"] is None: + @model_validator(mode="after") + def initial_chunk_if_historic_frequency_greater_than_0(self) -> Self: + if self.historic_frequency: + if self.historic_frequency > 0 and self.initial_chunk is None: raise ValueError( "Must include initial_chunk when historic_frequency > 0" ) - return values + return self From c1c1589c674809c6a7f51af9358c0be8b6fc7f9a Mon Sep 17 00:00:00 2001 From: Jamison French Date: Thu, 11 Jul 2024 16:58:03 -0500 Subject: [PATCH 4/7] fix: correct secret --- stactools_pipelines/pipelines/amazonia_1/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stactools_pipelines/pipelines/amazonia_1/config.yaml b/stactools_pipelines/pipelines/amazonia_1/config.yaml index 826c654..d0f5a01 100644 --- a/stactools_pipelines/pipelines/amazonia_1/config.yaml +++ b/stactools_pipelines/pipelines/amazonia_1/config.yaml @@ -1,7 +1,7 @@ --- id: "amazonia_1" compute: "awslambda" - secret_arn: "arn:aws:secretsmanager:eu-central-1:751953039297:secret:asdi-auth-stack-alukach/asdi-workflows-6awSos" + secret_arn: "arn:aws:secretsmanager:us-west-2:751953039297:secret:asdi-auth-stack-dev/asdi-workflows-fSzAw7" ingestor_url: "https://crtx3178aa.execute-api.us-west-2.amazonaws.com/dev" sns: "arn:aws:sns:us-west-2:599544552497:NewAM1Quicklook" # inventory_location: "" From cef26ea0928f0e2cea29a18bd4799fb55950b5d4 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Thu, 11 Jul 2024 21:59:05 -0500 Subject: [PATCH 5/7] chore: update secret --- .../pipelines/aws_noaa_oisst_avhrr_only/config.yaml | 2 +- stactools_pipelines/pipelines/cop_dem_30/config.yaml | 2 +- stactools_pipelines/pipelines/noaa_oisst/config.yaml | 2 +- stactools_pipelines/pipelines/sentinel1/config.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/config.yaml b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/config.yaml index 1429b6c..66ecfce 100644 --- a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/config.yaml +++ b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/config.yaml @@ -1,5 +1,5 @@ --- id: "aws_noaa_oisst_avhrr_only" compute: "awslambda" - secret_arn: "arn:aws:secretsmanager:eu-central-1:751953039297:secret:asdi-auth-stack-alukach/asdi-workflows-6awSos" + secret_arn: "arn:aws:secretsmanager:us-west-2:751953039297:secret:asdi-auth-stack-dev/asdi-workflows-fSzAw7" ingestor_url: "https://crtx3178aa.execute-api.us-west-2.amazonaws.com/dev" diff --git a/stactools_pipelines/pipelines/cop_dem_30/config.yaml b/stactools_pipelines/pipelines/cop_dem_30/config.yaml index 0fc5961..e16f98d 100644 --- a/stactools_pipelines/pipelines/cop_dem_30/config.yaml +++ b/stactools_pipelines/pipelines/cop_dem_30/config.yaml @@ -1,7 +1,7 @@ --- id: "cop_dem_30" compute: "awslambda" - secret_arn: "arn:aws:secretsmanager:eu-central-1:751953039297:secret:asdi-auth-stack-alukach/asdi-workflows-6awSos" + secret_arn: "arn:aws:secretsmanager:eu-central-1:751953039297:secret:asdi-auth-stack-dev/asdi-workflows-fSzAw7" ingestor_url: "https://crtx3178aa.execute-api.us-west-2.amazonaws.com/dev" inventory_location: "s3://asdi-athena-testing-eu-central-1/inventories/copernicus-dem-30m.csv" historic_frequency: 0 diff --git a/stactools_pipelines/pipelines/noaa_oisst/config.yaml b/stactools_pipelines/pipelines/noaa_oisst/config.yaml index 6ca688c..311b4e1 100644 --- a/stactools_pipelines/pipelines/noaa_oisst/config.yaml +++ b/stactools_pipelines/pipelines/noaa_oisst/config.yaml @@ -1,7 +1,7 @@ --- id: "noaa_oisst" compute: "awslambda" - secret_arn: "arn:aws:secretsmanager:eu-central-1:751953039297:secret:asdi-auth-stack-alukach/asdi-workflows-6awSos" + secret_arn: "arn:aws:secretsmanager:us-west-2:751953039297:secret:asdi-auth-stack-dev/asdi-workflows-fSzAw7" ingestor_url: "https://crtx3178aa.execute-api.us-west-2.amazonaws.com/dev" sns: "arn:aws:sns:us-east-1:709902155096:NewCDRSeaSurfTempOptInterpObject" inventory_location: "s3://nodd-bucket-inventory/noaa-cdr-sea-surface-temp-optimum-interpolation-pds/DailyCSV/hive/dt=2023-04-24-01-00/" diff --git a/stactools_pipelines/pipelines/sentinel1/config.yaml b/stactools_pipelines/pipelines/sentinel1/config.yaml index d047416..6d9044f 100644 --- a/stactools_pipelines/pipelines/sentinel1/config.yaml +++ b/stactools_pipelines/pipelines/sentinel1/config.yaml @@ -1,7 +1,7 @@ --- id: "sentinel1" compute: "awslambda" - secret_arn: "arn:aws:secretsmanager:eu-central-1:751953039297:secret:asdi-auth-stack-alukach/asdi-workflows-6awSos" + secret_arn: "arn:aws:secretsmanager:eu-central-1:751953039297:secret:asdi-auth-stack-dev/asdi-workflows-fSzAw7" ingestor_url: "https://crtx3178aa.execute-api.us-west-2.amazonaws.com/dev" sns: "arn:aws:sns:eu-central-1:214830741341:SentinelS1L1C" inventory_location: "s3://sentinel-inventory/sentinel-s1-l1c/sentinel-s1-l1c-inventory/hive/dt=2022-11-16-01-00/" From fd5e6c14c531b83bb90bbf5baa91b3aa546eb4ff Mon Sep 17 00:00:00 2001 From: Jamison French Date: Sun, 14 Jul 2024 04:03:36 -0500 Subject: [PATCH 6/7] chore: update default node runtime --- setup.py | 4 ++-- tox.ini | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 5ccf59d..c21bfcc 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,8 @@ from setuptools import find_packages, setup # type: ignore aws_cdk_extras = [ - "aws-cdk-lib==2.45.0", - "aws-cdk.aws-lambda-python-alpha==2.45.0a0", + "aws-cdk-lib==2.94.0", + "aws-cdk.aws-lambda-python-alpha==2.94.0a0", "constructs>=10.0.0", ] diff --git a/tox.ini b/tox.ini index 1b83f27..7aa3c66 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ passenv = whitelist_externals = sh commands = nodeenv --node=18.0.0 -p - npm install -g aws-cdk@2.45.0 + npm install -g aws-cdk@2.94.0 cdk --version pip install -e . From f6a200c52cb812751c95ee8493babf05366a814a Mon Sep 17 00:00:00 2001 From: Jamison French Date: Mon, 15 Jul 2024 13:14:12 -0500 Subject: [PATCH 7/7] chore: flake8 changes - updated --- .../pipelines/aws_noaa_oisst_avhrr_only/test_app.py | 2 +- .../pipelines/aws_noaa_oisst_avhrr_only/test_collection.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_app.py b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_app.py index a83a952..f438776 100644 --- a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_app.py +++ b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_app.py @@ -12,7 +12,7 @@ def test_create_item(): item = module_create_item() - assert type(item) is pystac.item.Item + assert isinstance(item, pystac.Item) @pytest.mark.parametrize("pipeline_id", ["aws_noaa_oisst_avhrr_only"]) diff --git a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_collection.py b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_collection.py index d544e70..2fce345 100644 --- a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_collection.py +++ b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_collection.py @@ -12,7 +12,7 @@ def test_create_collection(): collection = module_create_collection() - assert type(collection) is pystac.collection.Collection + assert isinstance(collection, pystac.Collection) @pytest.mark.parametrize("pipeline_id", ["aws_noaa_oisst_avhrr_only"])