diff --git a/setup.py b/setup.py index 5ccf59d..c21bfcc 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,8 @@ from setuptools import find_packages, setup # type: ignore aws_cdk_extras = [ - "aws-cdk-lib==2.45.0", - "aws-cdk.aws-lambda-python-alpha==2.45.0a0", + "aws-cdk-lib==2.94.0", + "aws-cdk.aws-lambda-python-alpha==2.94.0a0", "constructs>=10.0.0", ] diff --git a/stactools_pipelines/models/pipeline.py b/stactools_pipelines/models/pipeline.py index 4359331..eb3a3b8 100644 --- a/stactools_pipelines/models/pipeline.py +++ b/stactools_pipelines/models/pipeline.py @@ -1,7 +1,8 @@ from enum import Enum from typing import Optional +from typing_extensions import Self -from pydantic import BaseModel, root_validator +from pydantic import BaseModel, model_validator class ComputeEnum(str, Enum): @@ -13,22 +14,22 @@ class Pipeline(BaseModel): compute: ComputeEnum secret_arn: str ingestor_url: str - sns: Optional[str] - inventory_location: Optional[str] - historic_frequency: Optional[int] - initial_chunk: Optional[str] + sns: Optional[str] = None + inventory_location: Optional[str] = None + historic_frequency: Optional[int] = None + initial_chunk: Optional[str] = None - @root_validator(pre=False) - def historic_frequency_if_inventory_location(cls, values): - if values["inventory_location"] and values["historic_frequency"] is None: + @model_validator(mode="after") + def historic_frequency_if_inventory_location(self) -> Self: + if self.inventory_location and self.historic_frequency is None: raise ValueError("Must include historic_frequency with inventory_location") - return values + return self - @root_validator(pre=False) - def initial_chunk_if_historic_frequency_greater_than_0(cls, values): - if values["historic_frequency"]: - if values["historic_frequency"] > 0 and values["initial_chunk"] is None: + @model_validator(mode="after") + def initial_chunk_if_historic_frequency_greater_than_0(self) -> Self: + if self.historic_frequency: + if self.historic_frequency > 0 and self.initial_chunk is None: raise ValueError( "Must include initial_chunk when historic_frequency > 0" ) - return values + return self diff --git a/stactools_pipelines/pipelines/amazonia_1/config.yaml b/stactools_pipelines/pipelines/amazonia_1/config.yaml index 59fde3a..d0f5a01 100644 --- a/stactools_pipelines/pipelines/amazonia_1/config.yaml +++ b/stactools_pipelines/pipelines/amazonia_1/config.yaml @@ -1,9 +1,9 @@ --- id: "amazonia_1" compute: "awslambda" - secret_arn: "arn:aws:secretsmanager:eu-central-1:751953039297:secret:asdi-auth-stack-alukach/asdi-workflows-6awSos" + secret_arn: "arn:aws:secretsmanager:us-west-2:751953039297:secret:asdi-auth-stack-dev/asdi-workflows-fSzAw7" ingestor_url: "https://crtx3178aa.execute-api.us-west-2.amazonaws.com/dev" - sns: "arn:aws:sns:us-east-1:599544552497:NewAMQuicklook" + sns: "arn:aws:sns:us-west-2:599544552497:NewAM1Quicklook" # inventory_location: "" # historic_frequency: 1 # initial_chunk: "" diff --git a/stactools_pipelines/pipelines/amazonia_1/requirements.txt b/stactools_pipelines/pipelines/amazonia_1/requirements.txt index 123ceae..dc524c0 100644 --- a/stactools_pipelines/pipelines/amazonia_1/requirements.txt +++ b/stactools_pipelines/pipelines/amazonia_1/requirements.txt @@ -1,4 +1,4 @@ aws-lambda-powertools jmespath -stactools-amazonia-1 >= 0.1.2 +stactools-amazonia-1 >= 0.1.3 s3fs diff --git a/stactools_pipelines/pipelines/amazonia_1/test_app.py b/stactools_pipelines/pipelines/amazonia_1/test_app.py index 4f587cc..6071fac 100644 --- a/stactools_pipelines/pipelines/amazonia_1/test_app.py +++ b/stactools_pipelines/pipelines/amazonia_1/test_app.py @@ -33,7 +33,7 @@ def test_xml_key_from_quicklook_key(): @pytest.mark.parametrize("pipeline_id", ["amazonia_1"]) @pytest.mark.parametrize("module", ["app"]) -@pytest.mark.parametrize("bucket", ["amazonia-pds"]) +@pytest.mark.parametrize("bucket", ["brazil-eosats"]) @pytest.mark.parametrize("key", [KEY]) def test_handler( mock_env, # pylint: disable=unused-argument @@ -46,7 +46,7 @@ def test_handler( handler(event=sqs_event, context={}) get_token.assert_called_once() create_item.assert_called_once_with( - asset_href="s3://amazonia-pds/AMAZONIA1/WFI/035/020/" + asset_href="s3://brazil-eosats/AMAZONIA1/WFI/035/020/" "AMAZONIA_1_WFI_20220814_035_020_L4/AMAZONIA_1_WFI_20220814_035_020_L4_BAND2.xml" ) requests.post.assert_called_once_with( diff --git a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/config.yaml b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/config.yaml index 1429b6c..66ecfce 100644 --- a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/config.yaml +++ b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/config.yaml @@ -1,5 +1,5 @@ --- id: "aws_noaa_oisst_avhrr_only" compute: "awslambda" - secret_arn: "arn:aws:secretsmanager:eu-central-1:751953039297:secret:asdi-auth-stack-alukach/asdi-workflows-6awSos" + secret_arn: "arn:aws:secretsmanager:us-west-2:751953039297:secret:asdi-auth-stack-dev/asdi-workflows-fSzAw7" ingestor_url: "https://crtx3178aa.execute-api.us-west-2.amazonaws.com/dev" diff --git a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_app.py b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_app.py index fcbd340..f438776 100644 --- a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_app.py +++ b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_app.py @@ -12,7 +12,7 @@ def test_create_item(): item = module_create_item() - assert type(item) == pystac.item.Item + assert isinstance(item, pystac.Item) @pytest.mark.parametrize("pipeline_id", ["aws_noaa_oisst_avhrr_only"]) diff --git a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_collection.py b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_collection.py index c4e1584..2fce345 100644 --- a/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_collection.py +++ b/stactools_pipelines/pipelines/aws_noaa_oisst_avhrr_only/test_collection.py @@ -12,7 +12,7 @@ def test_create_collection(): collection = module_create_collection() - assert type(collection) == pystac.collection.Collection + assert isinstance(collection, pystac.Collection) @pytest.mark.parametrize("pipeline_id", ["aws_noaa_oisst_avhrr_only"]) diff --git a/stactools_pipelines/pipelines/cop_dem_30/config.yaml b/stactools_pipelines/pipelines/cop_dem_30/config.yaml index 0fc5961..e16f98d 100644 --- a/stactools_pipelines/pipelines/cop_dem_30/config.yaml +++ b/stactools_pipelines/pipelines/cop_dem_30/config.yaml @@ -1,7 +1,7 @@ --- id: "cop_dem_30" compute: "awslambda" - secret_arn: "arn:aws:secretsmanager:eu-central-1:751953039297:secret:asdi-auth-stack-alukach/asdi-workflows-6awSos" + secret_arn: "arn:aws:secretsmanager:eu-central-1:751953039297:secret:asdi-auth-stack-dev/asdi-workflows-fSzAw7" ingestor_url: "https://crtx3178aa.execute-api.us-west-2.amazonaws.com/dev" inventory_location: "s3://asdi-athena-testing-eu-central-1/inventories/copernicus-dem-30m.csv" historic_frequency: 0 diff --git a/stactools_pipelines/pipelines/noaa_oisst/config.yaml b/stactools_pipelines/pipelines/noaa_oisst/config.yaml index 6ca688c..311b4e1 100644 --- a/stactools_pipelines/pipelines/noaa_oisst/config.yaml +++ b/stactools_pipelines/pipelines/noaa_oisst/config.yaml @@ -1,7 +1,7 @@ --- id: "noaa_oisst" compute: "awslambda" - secret_arn: "arn:aws:secretsmanager:eu-central-1:751953039297:secret:asdi-auth-stack-alukach/asdi-workflows-6awSos" + secret_arn: "arn:aws:secretsmanager:us-west-2:751953039297:secret:asdi-auth-stack-dev/asdi-workflows-fSzAw7" ingestor_url: "https://crtx3178aa.execute-api.us-west-2.amazonaws.com/dev" sns: "arn:aws:sns:us-east-1:709902155096:NewCDRSeaSurfTempOptInterpObject" inventory_location: "s3://nodd-bucket-inventory/noaa-cdr-sea-surface-temp-optimum-interpolation-pds/DailyCSV/hive/dt=2023-04-24-01-00/" diff --git a/stactools_pipelines/pipelines/sentinel1/config.yaml b/stactools_pipelines/pipelines/sentinel1/config.yaml index d047416..6d9044f 100644 --- a/stactools_pipelines/pipelines/sentinel1/config.yaml +++ b/stactools_pipelines/pipelines/sentinel1/config.yaml @@ -1,7 +1,7 @@ --- id: "sentinel1" compute: "awslambda" - secret_arn: "arn:aws:secretsmanager:eu-central-1:751953039297:secret:asdi-auth-stack-alukach/asdi-workflows-6awSos" + secret_arn: "arn:aws:secretsmanager:eu-central-1:751953039297:secret:asdi-auth-stack-dev/asdi-workflows-fSzAw7" ingestor_url: "https://crtx3178aa.execute-api.us-west-2.amazonaws.com/dev" sns: "arn:aws:sns:eu-central-1:214830741341:SentinelS1L1C" inventory_location: "s3://sentinel-inventory/sentinel-s1-l1c/sentinel-s1-l1c-inventory/hive/dt=2022-11-16-01-00/" diff --git a/tox.ini b/tox.ini index 1b83f27..7aa3c66 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ passenv = whitelist_externals = sh commands = nodeenv --node=18.0.0 -p - npm install -g aws-cdk@2.45.0 + npm install -g aws-cdk@2.94.0 cdk --version pip install -e .