From f382c5aaaef78dcc8cfecb1fb1f1cb872c44aeaa Mon Sep 17 00:00:00 2001 From: elay Date: Thu, 6 Jun 2024 16:30:32 -0700 Subject: [PATCH 01/41] init --- pcfuncs/ipban/__init__.py | 27 ++++++++++++ pcfuncs/ipban/constants.py | 10 +++++ pcfuncs/ipban/function.json | 11 +++++ pcfuncs/ipban/models.py | 69 +++++++++++++++++++++++++++++ pcfuncs/ipban/readme.md | 11 +++++ pcfuncs/tests/ipban/__init__.py | 0 pcfuncs/tests/ipban/test_ipban.py | 73 +++++++++++++++++++++++++++++++ 7 files changed, 201 insertions(+) create mode 100644 pcfuncs/ipban/__init__.py create mode 100644 pcfuncs/ipban/constants.py create mode 100644 pcfuncs/ipban/function.json create mode 100644 pcfuncs/ipban/models.py create mode 100644 pcfuncs/ipban/readme.md create mode 100644 pcfuncs/tests/ipban/__init__.py create mode 100644 pcfuncs/tests/ipban/test_ipban.py diff --git a/pcfuncs/ipban/__init__.py b/pcfuncs/ipban/__init__.py new file mode 100644 index 00000000..ee9292fc --- /dev/null +++ b/pcfuncs/ipban/__init__.py @@ -0,0 +1,27 @@ +import datetime +import logging +import os + +import azure.functions as func +from azure.data.tables import TableClient, TableServiceClient, UpdateMode +from azure.identity import DefaultAzureCredential +from azure.monitor.query import LogsQueryClient +from .constants import * +from .models import UpdateBannedIPTask + + +def main(mytimer: func.TimerRequest) -> None: + utc_timestamp = ( + datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat() + ) + logging.info("Python timer trigger function ran at %s", utc_timestamp) + credential: DefaultAzureCredential = DefaultAzureCredential() + logs_query_client: LogsQueryClient = LogsQueryClient(credential) + table_service_client: TableServiceClient = TableServiceClient( + endpoint=STORAGE_ACCOUNT_URL, credential=credential + ) + table_client: TableClient = table_service_client.create_table_if_not_exists( + BANNED_IP_TABLE + ) + task: UpdateBannedIPTask = UpdateBannedIPTask(logs_query_client, table_client) + task.run() diff --git a/pcfuncs/ipban/constants.py b/pcfuncs/ipban/constants.py new file mode 100644 index 00000000..11563dd9 --- /dev/null +++ b/pcfuncs/ipban/constants.py @@ -0,0 +1,10 @@ +# Constants related to Azure Table Storage +STORAGE_ACCOUNT_URL = "https://spatiocitest.table.core.windows.net/" +BANNED_IP_TABLE = "blobStorageBannedIp" + +# Log Analytics Workspace constants +LOG_ANALYTICS_WORKSPACE_ID = "78d48390-b6bb-49a9-b7fd-a86f6522e9c4" + +# Time and threshold settings +TIME_WINDOW_IN_HOURS = 24 +THRESHOLD_READ_COUNT_IN_GB = 100 diff --git a/pcfuncs/ipban/function.json b/pcfuncs/ipban/function.json new file mode 100644 index 00000000..38cb7eef --- /dev/null +++ b/pcfuncs/ipban/function.json @@ -0,0 +1,11 @@ +{ + "scriptFile": "__init__.py", + "bindings": [ + { + "name": "mytimer", + "type": "timerTrigger", + "direction": "in", + "schedule": "0 */1 * * * *" + } + ] +} \ No newline at end of file diff --git a/pcfuncs/ipban/models.py b/pcfuncs/ipban/models.py new file mode 100644 index 00000000..d5053fb1 --- /dev/null +++ b/pcfuncs/ipban/models.py @@ -0,0 +1,69 @@ +import datetime +import logging + +import azure.functions as func +from azure.data.tables import TableClient, TableServiceClient, UpdateMode +from azure.identity import DefaultAzureCredential +from azure.monitor.query import LogsQueryClient +from .constants import * + + +class UpdateBannedIPTask: + def __init__( + self, + logs_query_client: LogsQueryClient, + table_client: TableClient, + ): + self.log_query_client = logs_query_client + self.table_client = table_client + + def run(self): + utc_timestamp = ( + datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat() + ) + logging.info("Python timer trigger function ran at %s", utc_timestamp) + query_result = self.get_blob_logs_query_result() + self.update_banned_ips(query_result) + + def get_blob_logs_query_result(self): + query = f""" + StorageBlobLogs + | where TimeGenerated > ago({TIME_WINDOW_IN_HOURS}h) + | extend IpAddress = tostring(split(CallerIpAddress, ":")[0]) + | summarize readcount = sum(ResponseBodySize) / (1024 * 1024 * 1024) by IpAddress + | where readcount > {THRESHOLD_READ_COUNT_IN_GB} + """ + response = self.log_query_client.query_workspace( + LOG_ANALYTICS_WORKSPACE_ID, query, timespan=None + ) + return response.tables[0].rows + + def update_banned_ips(self, query_result): + existing_ips = { + entity["RowKey"]: entity for entity in self.table_client.list_entities() + } + print(existing_ips) + result_ips = set() + for result in query_result: + ip_address, read_count = result[0], int(result[1]) + result_ips.add(ip_address) + entity = { + "PartitionKey": ip_address, + "RowKey": ip_address, + "ReadCount": read_count, + "Threshold": THRESHOLD_READ_COUNT_IN_GB, + "TimeWindow": TIME_WINDOW_IN_HOURS, + } + + if ip_address in existing_ips: + self.table_client.update_entity(entity, mode=UpdateMode.REPLACE) + else: + self.table_client.create_entity(entity) + + for ip_address in existing_ips: + if ip_address not in result_ips: + self.table_client.delete_entity( + partition_key=ip_address, row_key=ip_address + ) + + logging.info("Table sync complete.") diff --git a/pcfuncs/ipban/readme.md b/pcfuncs/ipban/readme.md new file mode 100644 index 00000000..e8b7e887 --- /dev/null +++ b/pcfuncs/ipban/readme.md @@ -0,0 +1,11 @@ +# TimerTrigger - Python + +The `TimerTrigger` makes it incredibly easy to have your functions executed on a schedule. This sample demonstrates a simple use case of calling your function every 5 minutes. + +## How it works + +For a `TimerTrigger` to work, you provide a schedule in the form of a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression)(See the link for full details). A cron expression is a string with 6 separate expressions which represent a given schedule via patterns. The pattern we use to represent every 5 minutes is `0 */5 * * * *`. This, in plain text, means: "When seconds is equal to 0, minutes is divisible by 5, for any hour, day of the month, month, day of the week, or year". + +## Learn more + + Documentation diff --git a/pcfuncs/tests/ipban/__init__.py b/pcfuncs/tests/ipban/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py new file mode 100644 index 00000000..8992bc86 --- /dev/null +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -0,0 +1,73 @@ +import pytest +from azure.data.tables import TableClient, TableServiceClient, UpdateMode +from ipban.constants import * +from ipban.models import UpdateBannedIPTask + +def print_table_entries(table_client: TableClient): + print(f"Printing all entries in the table: {table_client.table_name}") + entities = table_client.list_entities() + for entity in entities: + print(entity) + +@pytest.fixture +def mock_clients(mocker): + mock_response = mocker.MagicMock() + expected_rows = [("192.168.1.1", 150)] + mock_response.tables[0].rows = expected_rows + mock_logs_query_client = mocker.MagicMock() + mock_logs_query_client.query_workspace.return_value = mock_response + + STORAGE_ACCOUNT_URL = "http://127.0.0.1:10002/devstoreaccount1" + STORAGE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + CONNECTION_STRING = f"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey={STORAGE_ACCOUNT_KEY};TableEndpoint={STORAGE_ACCOUNT_URL};" + + # Use Azurite for unit tests and populate the table with initial data + table_service = TableServiceClient.from_connection_string(CONNECTION_STRING) + table_client = table_service.create_table_if_not_exists(table_name=BANNED_IP_TABLE) + entities = [ + { + "PartitionKey": "192.168.1.1", + "RowKey": "192.168.1.1", + "ReadCount": 50, + "Threshold": 100, + "TimeWindow": 15, + }, + { + "PartitionKey": "192.168.1.2", + "RowKey": "192.168.1.2", + "ReadCount": 150, + "Threshold": 150, + "TimeWindow": 30, + }, + ] + for entity in entities: + table_client.create_entity(entity) + print("Populating the table") + print_table_entries(table_client) + yield mock_logs_query_client, table_client + + print("After tests are completed") + print_table_entries(table_client) + # Clear all entities from the table + entities = list(table_client.list_entities()) + for entity in entities: + table_client.delete_entity( + partition_key=entity["PartitionKey"], row_key=entity["RowKey"] + ) + + print("After cleanup are completed") + print_table_entries(table_client) + + +def test_update_banned_ip_task(mock_clients): + mock_logs_query_client, table_client = mock_clients + task: UpdateBannedIPTask = UpdateBannedIPTask(mock_logs_query_client, table_client) + task.run() + print("Test is done:") + print_table_entries(table_client) + + # Fetch updated data from table to check assertions + # updated_entity_1 = table_client.get_entity("192.168.1.1", "192.168.1.1") + # updated_entity_3 = table_client.get_entity("192.168.1.3", "192.168.1.3") + # assert updated_entity_1["ReadCount"] == 200 + # assert updated_entity_3["ReadCount"] == 300 From 64704145ed05e9e6648c1d05fa806a0e8e8e5448 Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 10 Jun 2024 19:46:46 -0700 Subject: [PATCH 02/41] update --- docker-compose.yml | 1 + pcfuncs/Dockerfile | 3 +- pcfuncs/ipban/__init__.py | 6 +- pcfuncs/ipban/constants.py | 2 +- pcfuncs/ipban/models.py | 36 ++++---- pcfuncs/requirements.txt | 8 +- pcfuncs/tests/conftest.py | 29 ++++++ pcfuncs/tests/ipban/test_ipban.py | 148 +++++++++++++++++++++--------- 8 files changed, 167 insertions(+), 66 deletions(-) create mode 100644 pcfuncs/tests/conftest.py diff --git a/docker-compose.yml b/docker-compose.yml index 63790237..c29121a6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,6 +53,7 @@ services: - ./pccommon:/home/site/pccommon - ./pcfuncs:/home/site/wwwroot - .:/opt/src + - ~/.azure:/home/.azure nginx: image: pc-apis-nginx diff --git a/pcfuncs/Dockerfile b/pcfuncs/Dockerfile index 3beb133f..888f90ce 100644 --- a/pcfuncs/Dockerfile +++ b/pcfuncs/Dockerfile @@ -1,7 +1,8 @@ -FROM mcr.microsoft.com/azure-functions/python:4-python3.8 +FROM mcr.microsoft.com/azure-functions/python:4-python3.10 # git required for pip installs from git RUN apt update && apt install -y git +RUN curl -sL https://aka.ms/InstallAzureCLIDeb | bash ENV AzureWebJobsScriptRoot=/home/site/wwwroot \ AzureFunctionsJobHost__Logging__Console__IsEnabled=true diff --git a/pcfuncs/ipban/__init__.py b/pcfuncs/ipban/__init__.py index ee9292fc..0b71a4f0 100644 --- a/pcfuncs/ipban/__init__.py +++ b/pcfuncs/ipban/__init__.py @@ -1,20 +1,20 @@ import datetime import logging -import os import azure.functions as func from azure.data.tables import TableClient, TableServiceClient, UpdateMode from azure.identity import DefaultAzureCredential from azure.monitor.query import LogsQueryClient + from .constants import * from .models import UpdateBannedIPTask def main(mytimer: func.TimerRequest) -> None: - utc_timestamp = ( + utc_timestamp: str = ( datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat() ) - logging.info("Python timer trigger function ran at %s", utc_timestamp) + logging.info("Updating the ip ban list at %s", utc_timestamp) credential: DefaultAzureCredential = DefaultAzureCredential() logs_query_client: LogsQueryClient = LogsQueryClient(credential) table_service_client: TableServiceClient = TableServiceClient( diff --git a/pcfuncs/ipban/constants.py b/pcfuncs/ipban/constants.py index 11563dd9..1e16bbd8 100644 --- a/pcfuncs/ipban/constants.py +++ b/pcfuncs/ipban/constants.py @@ -2,7 +2,7 @@ STORAGE_ACCOUNT_URL = "https://spatiocitest.table.core.windows.net/" BANNED_IP_TABLE = "blobStorageBannedIp" -# Log Analytics Workspace constants +# Log Analytics Workspace: pc-api-loganalytics LOG_ANALYTICS_WORKSPACE_ID = "78d48390-b6bb-49a9-b7fd-a86f6522e9c4" # Time and threshold settings diff --git a/pcfuncs/ipban/models.py b/pcfuncs/ipban/models.py index d5053fb1..e890dcba 100644 --- a/pcfuncs/ipban/models.py +++ b/pcfuncs/ipban/models.py @@ -1,10 +1,15 @@ import datetime import logging +from typing import Any, List, Set import azure.functions as func from azure.data.tables import TableClient, TableServiceClient, UpdateMode from azure.identity import DefaultAzureCredential from azure.monitor.query import LogsQueryClient +from azure.monitor.query._models import ( + LogsTableRow, +) + from .constants import * @@ -13,39 +18,36 @@ def __init__( self, logs_query_client: LogsQueryClient, table_client: TableClient, - ): - self.log_query_client = logs_query_client - self.table_client = table_client + ) -> None: + self.log_query_client: LogsQueryClient = logs_query_client + self.table_client: TableClient = table_client - def run(self): - utc_timestamp = ( - datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat() - ) - logging.info("Python timer trigger function ran at %s", utc_timestamp) - query_result = self.get_blob_logs_query_result() + def run(self) -> List[LogsTableRow]: + query_result: List[LogsTableRow] = self.get_blob_logs_query_result() self.update_banned_ips(query_result) + return query_result - def get_blob_logs_query_result(self): - query = f""" + def get_blob_logs_query_result(self) -> List[LogsTableRow]: + query: str = f""" StorageBlobLogs | where TimeGenerated > ago({TIME_WINDOW_IN_HOURS}h) | extend IpAddress = tostring(split(CallerIpAddress, ":")[0]) | summarize readcount = sum(ResponseBodySize) / (1024 * 1024 * 1024) by IpAddress | where readcount > {THRESHOLD_READ_COUNT_IN_GB} """ - response = self.log_query_client.query_workspace( + response: Any = self.log_query_client.query_workspace( LOG_ANALYTICS_WORKSPACE_ID, query, timespan=None ) return response.tables[0].rows - def update_banned_ips(self, query_result): + def update_banned_ips(self, query_result: List[LogsTableRow]) -> None: existing_ips = { entity["RowKey"]: entity for entity in self.table_client.list_entities() } - print(existing_ips) - result_ips = set() + result_ips: Set[str] = set() for result in query_result: - ip_address, read_count = result[0], int(result[1]) + ip_address: str = result[0] + read_count: int = int(result[1]) result_ips.add(ip_address) entity = { "PartitionKey": ip_address, @@ -66,4 +68,4 @@ def update_banned_ips(self, query_result): partition_key=ip_address, row_key=ip_address ) - logging.info("Table sync complete.") + logging.info("IP ban list has been updated successfully") diff --git a/pcfuncs/requirements.txt b/pcfuncs/requirements.txt index 5d939459..dc645a9f 100644 --- a/pcfuncs/requirements.txt +++ b/pcfuncs/requirements.txt @@ -14,7 +14,13 @@ pillow==10.3.0 pyproj==3.3.1 pydantic>=1.9,<2.0.0 rasterio==1.3.* - +azure-core==1.30.1 +azure-data-tables==12.4.0 +azure-identity==1.7.1 +azure-monitor-query==1.3.0 +azure-storage-blob==12.20.0 +opencensus-ext-azure==1.0.8 +pytest-mock==3.14.0 # Deployment needs to copy the local code into # the app code directory, so requires a separate # requirements file. diff --git a/pcfuncs/tests/conftest.py b/pcfuncs/tests/conftest.py new file mode 100644 index 00000000..398fc9b2 --- /dev/null +++ b/pcfuncs/tests/conftest.py @@ -0,0 +1,29 @@ +from typing import List + +import pytest + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--no-integration", + action="store_true", + default=False, + help="don't run integration tests", + ) + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line("markers", "integration: mark as an integration test") + + +def pytest_collection_modifyitems( + config: pytest.Config, items: List[pytest.Item] +) -> None: + if config.getoption("--no-integration"): + # --no-integration given in cli: skip integration tests + skip_integration = pytest.mark.skip( + reason="needs --no-integration option to run" + ) + for item in items: + if "integration" in item.keywords: + item.add_marker(skip_integration) diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index 8992bc86..6b10690d 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -1,73 +1,135 @@ +from html import entities +from typing import Any, Dict, Generator, List, Literal, Tuple +from unittest.mock import MagicMock + import pytest +from azure.core.paging import ItemPaged from azure.data.tables import TableClient, TableServiceClient, UpdateMode +from azure.data.tables._entity import TableEntity +from azure.identity import DefaultAzureCredential +from azure.monitor.query import LogsQueryClient +from azure.monitor.query._models import ( + LogsTableRow, +) from ipban.constants import * from ipban.models import UpdateBannedIPTask +from pytest_mock import MockerFixture -def print_table_entries(table_client: TableClient): - print(f"Printing all entries in the table: {table_client.table_name}") - entities = table_client.list_entities() - for entity in entities: - print(entity) - -@pytest.fixture -def mock_clients(mocker): - mock_response = mocker.MagicMock() - expected_rows = [("192.168.1.1", 150)] - mock_response.tables[0].rows = expected_rows - mock_logs_query_client = mocker.MagicMock() - mock_logs_query_client.query_workspace.return_value = mock_response +MOCK_LOGS_QUERY_RESULT = [("192.168.1.1", 170), ("192.168.1.4", 420)] +TEST_BANNED_IP_TABLE = "testBlobStorageBannedIp" - STORAGE_ACCOUNT_URL = "http://127.0.0.1:10002/devstoreaccount1" - STORAGE_ACCOUNT_KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" - CONNECTION_STRING = f"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey={STORAGE_ACCOUNT_KEY};TableEndpoint={STORAGE_ACCOUNT_URL};" - # Use Azurite for unit tests and populate the table with initial data - table_service = TableServiceClient.from_connection_string(CONNECTION_STRING) - table_client = table_service.create_table_if_not_exists(table_name=BANNED_IP_TABLE) - entities = [ +def populate_banned_ip_table(table_client: TableClient) -> List[Dict[str, Any]]: + print("Populating the table") + entities: List[Dict[str, Any]] = [ { "PartitionKey": "192.168.1.1", "RowKey": "192.168.1.1", - "ReadCount": 50, - "Threshold": 100, - "TimeWindow": 15, + "ReadCount": 647, + "Threshold": THRESHOLD_READ_COUNT_IN_GB, + "TimeWindow": TIME_WINDOW_IN_HOURS, }, { "PartitionKey": "192.168.1.2", "RowKey": "192.168.1.2", - "ReadCount": 150, - "Threshold": 150, - "TimeWindow": 30, + "ReadCount": 214, + "Threshold": THRESHOLD_READ_COUNT_IN_GB, + "TimeWindow": TIME_WINDOW_IN_HOURS, + }, + { + "PartitionKey": "192.168.1.3", + "RowKey": "192.168.1.3", + "ReadCount": 550, + "Threshold": THRESHOLD_READ_COUNT_IN_GB, + "TimeWindow": TIME_WINDOW_IN_HOURS, }, ] for entity in entities: table_client.create_entity(entity) - print("Populating the table") - print_table_entries(table_client) - yield mock_logs_query_client, table_client + return entities - print("After tests are completed") - print_table_entries(table_client) - # Clear all entities from the table + +def clear_table(table_client: TableClient) -> None: entities = list(table_client.list_entities()) for entity in entities: table_client.delete_entity( partition_key=entity["PartitionKey"], row_key=entity["RowKey"] ) + entities = list(table_client.list_entities()) + assert len(entities) == 0 + - print("After cleanup are completed") - print_table_entries(table_client) +@pytest.fixture +def mock_clients( + mocker: MockerFixture, +) -> Generator[Tuple[MagicMock, TableClient], Any, None]: + mock_response: MagicMock = mocker.MagicMock() + mock_response.tables[0].rows = MOCK_LOGS_QUERY_RESULT + logs_query_client: MagicMock = mocker.MagicMock() + logs_query_client.query_workspace.return_value = mock_response + CONNECTION_STRING: str = f"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;TableEndpoint=http://azurite:10002/devstoreaccount1;" + # Use Azurite for unit tests and populate the table with initial data + table_service: TableServiceClient = TableServiceClient.from_connection_string( + CONNECTION_STRING + ) + table_client = table_service.create_table_if_not_exists( + table_name=TEST_BANNED_IP_TABLE + ) + # Pre-populate the banned ip table + populate_banned_ip_table(table_client) + yield logs_query_client, table_client + + # Clear all entities from the table + clear_table(table_client) -def test_update_banned_ip_task(mock_clients): + +@pytest.fixture +def integration_clients( + mocker: MockerFixture, +) -> Generator[Tuple[LogsQueryClient, TableClient], Any, None]: + credential: DefaultAzureCredential = DefaultAzureCredential() + logs_query_client: LogsQueryClient = LogsQueryClient(credential) + table_service_client: TableServiceClient = TableServiceClient( + endpoint=STORAGE_ACCOUNT_URL, credential=credential + ) + table_client: TableClient = table_service_client.create_table_if_not_exists( + TEST_BANNED_IP_TABLE + ) + # Pre-populate the banned ip table + populate_banned_ip_table(table_client) + yield logs_query_client, table_client + # Clear all entities from the table + clear_table(table_client) + + +@pytest.mark.integration +def test_update_banned_ip_integration( + integration_clients: Tuple[LogsQueryClient, TableClient] +) -> None: + print("Integration test is running") + logs_query_client, table_client = integration_clients + task: UpdateBannedIPTask = UpdateBannedIPTask(logs_query_client, table_client) + # retrieve the logs query result from pc-api-loganalytics + logs_query_result: List[LogsTableRow] = task.run() + entities = list(table_client.list_entities()) + assert len(logs_query_result) == len(entities) + for ip, expected_read_count in logs_query_result: + entity = table_client.get_entity(ip, ip) + assert entity["ReadCount"] == expected_read_count + assert entity["Threshold"] == THRESHOLD_READ_COUNT_IN_GB + assert entity["TimeWindow"] == TIME_WINDOW_IN_HOURS + + +def test_update_banned_ip(mock_clients: Tuple[MagicMock, TableClient]) -> None: + print("Unit test is running") mock_logs_query_client, table_client = mock_clients task: UpdateBannedIPTask = UpdateBannedIPTask(mock_logs_query_client, table_client) task.run() - print("Test is done:") - print_table_entries(table_client) - - # Fetch updated data from table to check assertions - # updated_entity_1 = table_client.get_entity("192.168.1.1", "192.168.1.1") - # updated_entity_3 = table_client.get_entity("192.168.1.3", "192.168.1.3") - # assert updated_entity_1["ReadCount"] == 200 - # assert updated_entity_3["ReadCount"] == 300 + entities = list(table_client.list_entities()) + assert len(entities) == len(MOCK_LOGS_QUERY_RESULT) + for ip, expected_read_count in MOCK_LOGS_QUERY_RESULT: + entity = table_client.get_entity(ip, ip) + assert entity["ReadCount"] == expected_read_count + assert entity["Threshold"] == THRESHOLD_READ_COUNT_IN_GB + assert entity["TimeWindow"] == TIME_WINDOW_IN_HOURS From ad60299a08a8f1f05131aafbc464f7be255f89a7 Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 10 Jun 2024 20:00:30 -0700 Subject: [PATCH 03/41] update flake8 config --- .flake8 | 3 ++- pcfuncs/ipban/function.json | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.flake8 b/.flake8 index 419d1104..e034c121 100644 --- a/.flake8 +++ b/.flake8 @@ -4,4 +4,5 @@ extend-ignore = E203, W503 exclude = .git __pycache__ - setup.py \ No newline at end of file + setup.py + .venv \ No newline at end of file diff --git a/pcfuncs/ipban/function.json b/pcfuncs/ipban/function.json index 38cb7eef..5f8080ed 100644 --- a/pcfuncs/ipban/function.json +++ b/pcfuncs/ipban/function.json @@ -5,7 +5,7 @@ "name": "mytimer", "type": "timerTrigger", "direction": "in", - "schedule": "0 */1 * * * *" + "schedule": "0 0 * * * *" } ] } \ No newline at end of file From 90a9724ece5474f974f9899ab36ea3908539778a Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 10 Jun 2024 20:08:53 -0700 Subject: [PATCH 04/41] code format changes --- pcfuncs/ipban/__init__.py | 5 +++-- pcfuncs/ipban/models.py | 11 ++++++----- pcfuncs/tests/ipban/test_ipban.py | 22 +++++++++++++--------- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/pcfuncs/ipban/__init__.py b/pcfuncs/ipban/__init__.py index 0b71a4f0..897cbf1d 100644 --- a/pcfuncs/ipban/__init__.py +++ b/pcfuncs/ipban/__init__.py @@ -2,11 +2,12 @@ import logging import azure.functions as func -from azure.data.tables import TableClient, TableServiceClient, UpdateMode +from azure.data.tables import TableClient, TableServiceClient from azure.identity import DefaultAzureCredential from azure.monitor.query import LogsQueryClient -from .constants import * +from constants import BANNED_IP_TABLE, STORAGE_ACCOUNT_URL + from .models import UpdateBannedIPTask diff --git a/pcfuncs/ipban/models.py b/pcfuncs/ipban/models.py index e890dcba..4842504b 100644 --- a/pcfuncs/ipban/models.py +++ b/pcfuncs/ipban/models.py @@ -1,16 +1,17 @@ -import datetime import logging from typing import Any, List, Set -import azure.functions as func -from azure.data.tables import TableClient, TableServiceClient, UpdateMode -from azure.identity import DefaultAzureCredential +from azure.data.tables import TableClient, UpdateMode from azure.monitor.query import LogsQueryClient from azure.monitor.query._models import ( LogsTableRow, ) -from .constants import * +from constants import ( + LOG_ANALYTICS_WORKSPACE_ID, + THRESHOLD_READ_COUNT_IN_GB, + TIME_WINDOW_IN_HOURS, +) class UpdateBannedIPTask: diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index 6b10690d..f1ea15bc 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -1,17 +1,19 @@ -from html import entities -from typing import Any, Dict, Generator, List, Literal, Tuple +from typing import Any, Dict, Generator, List, Tuple from unittest.mock import MagicMock -import pytest -from azure.core.paging import ItemPaged -from azure.data.tables import TableClient, TableServiceClient, UpdateMode from azure.data.tables._entity import TableEntity +import pytest +from azure.data.tables import TableClient, TableServiceClient from azure.identity import DefaultAzureCredential from azure.monitor.query import LogsQueryClient from azure.monitor.query._models import ( LogsTableRow, ) -from ipban.constants import * +from constants import ( + STORAGE_ACCOUNT_URL, + THRESHOLD_READ_COUNT_IN_GB, + TIME_WINDOW_IN_HOURS, +) from ipban.models import UpdateBannedIPTask from pytest_mock import MockerFixture @@ -67,12 +69,14 @@ def mock_clients( mock_response.tables[0].rows = MOCK_LOGS_QUERY_RESULT logs_query_client: MagicMock = mocker.MagicMock() logs_query_client.query_workspace.return_value = mock_response - CONNECTION_STRING: str = f"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;TableEndpoint=http://azurite:10002/devstoreaccount1;" + CONNECTION_STRING: str = ( + "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;TableEndpoint=http://azurite:10002/devstoreaccount1;" + ) # Use Azurite for unit tests and populate the table with initial data table_service: TableServiceClient = TableServiceClient.from_connection_string( CONNECTION_STRING ) - table_client = table_service.create_table_if_not_exists( + table_client: TableClient = table_service.create_table_if_not_exists( table_name=TEST_BANNED_IP_TABLE ) @@ -115,7 +119,7 @@ def test_update_banned_ip_integration( entities = list(table_client.list_entities()) assert len(logs_query_result) == len(entities) for ip, expected_read_count in logs_query_result: - entity = table_client.get_entity(ip, ip) + entity: TableEntity = table_client.get_entity(ip, ip) assert entity["ReadCount"] == expected_read_count assert entity["Threshold"] == THRESHOLD_READ_COUNT_IN_GB assert entity["TimeWindow"] == TIME_WINDOW_IN_HOURS From 9a99dcc0b420451e6cd9507dc88b707d31934350 Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 10 Jun 2024 20:21:24 -0700 Subject: [PATCH 05/41] fix long lines --- pcfuncs/ipban/models.py | 3 ++- pcfuncs/tests/ipban/test_ipban.py | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pcfuncs/ipban/models.py b/pcfuncs/ipban/models.py index 4842504b..a5e65f1f 100644 --- a/pcfuncs/ipban/models.py +++ b/pcfuncs/ipban/models.py @@ -33,7 +33,8 @@ def get_blob_logs_query_result(self) -> List[LogsTableRow]: StorageBlobLogs | where TimeGenerated > ago({TIME_WINDOW_IN_HOURS}h) | extend IpAddress = tostring(split(CallerIpAddress, ":")[0]) - | summarize readcount = sum(ResponseBodySize) / (1024 * 1024 * 1024) by IpAddress + | summarize readcount = sum(ResponseBodySize) / (1024 * 1024 * 1024) + by IpAddress | where readcount > {THRESHOLD_READ_COUNT_IN_GB} """ response: Any = self.log_query_client.query_workspace( diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index f1ea15bc..94902527 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -70,7 +70,10 @@ def mock_clients( logs_query_client: MagicMock = mocker.MagicMock() logs_query_client.query_workspace.return_value = mock_response CONNECTION_STRING: str = ( - "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;TableEndpoint=http://azurite:10002/devstoreaccount1;" + "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" + "AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu" + "Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" + "TableEndpoint=http://azurite:10002/devstoreaccount1;" ) # Use Azurite for unit tests and populate the table with initial data table_service: TableServiceClient = TableServiceClient.from_connection_string( From 55ad5d97bae4783e8dd5446b1425b5f1e1b1a3af Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 10 Jun 2024 20:25:09 -0700 Subject: [PATCH 06/41] fix imports --- pcfuncs/ipban/__init__.py | 2 +- pcfuncs/ipban/models.py | 2 +- pcfuncs/tests/ipban/test_ipban.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pcfuncs/ipban/__init__.py b/pcfuncs/ipban/__init__.py index 897cbf1d..bf04405a 100644 --- a/pcfuncs/ipban/__init__.py +++ b/pcfuncs/ipban/__init__.py @@ -6,7 +6,7 @@ from azure.identity import DefaultAzureCredential from azure.monitor.query import LogsQueryClient -from constants import BANNED_IP_TABLE, STORAGE_ACCOUNT_URL +from .constants import BANNED_IP_TABLE, STORAGE_ACCOUNT_URL from .models import UpdateBannedIPTask diff --git a/pcfuncs/ipban/models.py b/pcfuncs/ipban/models.py index a5e65f1f..af80e788 100644 --- a/pcfuncs/ipban/models.py +++ b/pcfuncs/ipban/models.py @@ -7,7 +7,7 @@ LogsTableRow, ) -from constants import ( +from .constants import ( LOG_ANALYTICS_WORKSPACE_ID, THRESHOLD_READ_COUNT_IN_GB, TIME_WINDOW_IN_HOURS, diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index 94902527..fef899e3 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -9,7 +9,7 @@ from azure.monitor.query._models import ( LogsTableRow, ) -from constants import ( +from ipban.constants import ( STORAGE_ACCOUNT_URL, THRESHOLD_READ_COUNT_IN_GB, TIME_WINDOW_IN_HOURS, From 3148e11c3f4e5c7eecd97a023ea1f2e936890b90 Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 10 Jun 2024 21:34:55 -0700 Subject: [PATCH 07/41] function app changes --- deployment/terraform/resources/functions.tf | 2 +- deployment/terraform/resources/storage_account.tf | 5 +++++ pcfuncs/ipban/constants.py | 4 ++-- pcfuncs/tests/ipban/test_ipban.py | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/deployment/terraform/resources/functions.tf b/deployment/terraform/resources/functions.tf index cdcf6ebf..f38ce3cb 100644 --- a/deployment/terraform/resources/functions.tf +++ b/deployment/terraform/resources/functions.tf @@ -46,7 +46,7 @@ resource "azurerm_function_app" "pcfuncs" { os_type = "linux" version = "~4" site_config { - linux_fx_version = "PYTHON|3.8" + linux_fx_version = "PYTHON|3.10" use_32_bit_worker_process = false ftps_state = "Disabled" diff --git a/deployment/terraform/resources/storage_account.tf b/deployment/terraform/resources/storage_account.tf index e55aa6d1..66f68432 100644 --- a/deployment/terraform/resources/storage_account.tf +++ b/deployment/terraform/resources/storage_account.tf @@ -24,3 +24,8 @@ resource "azurerm_storage_table" "ipexceptionlist" { name = "ipexceptionlist" storage_account_name = azurerm_storage_account.pc.name } + +resource "azurerm_storage_table" "blobstoragebannedip" { + name = "blobstoragebannedip" + storage_account_name = azurerm_storage_account.pc.name +} \ No newline at end of file diff --git a/pcfuncs/ipban/constants.py b/pcfuncs/ipban/constants.py index 1e16bbd8..812fd365 100644 --- a/pcfuncs/ipban/constants.py +++ b/pcfuncs/ipban/constants.py @@ -1,6 +1,6 @@ # Constants related to Azure Table Storage -STORAGE_ACCOUNT_URL = "https://spatiocitest.table.core.windows.net/" -BANNED_IP_TABLE = "blobStorageBannedIp" +STORAGE_ACCOUNT_URL = "https://pcfilestest.table.core.windows.net/" +BANNED_IP_TABLE = "blobstoragebannedip" # Log Analytics Workspace: pc-api-loganalytics LOG_ANALYTICS_WORKSPACE_ID = "78d48390-b6bb-49a9-b7fd-a86f6522e9c4" diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index fef899e3..33c2bc50 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -18,7 +18,7 @@ from pytest_mock import MockerFixture MOCK_LOGS_QUERY_RESULT = [("192.168.1.1", 170), ("192.168.1.4", 420)] -TEST_BANNED_IP_TABLE = "testBlobStorageBannedIp" +TEST_BANNED_IP_TABLE = "testblobstoragebannedip" def populate_banned_ip_table(table_client: TableClient) -> List[Dict[str, Any]]: From bf2c7cc796c69c19103b227172f8645956ea7728 Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 10 Jun 2024 21:38:51 -0700 Subject: [PATCH 08/41] code format changes --- deployment/terraform/resources/storage_account.tf | 5 +++++ pcfuncs/ipban/__init__.py | 1 - pcfuncs/ipban/models.py | 4 +--- pcfuncs/tests/ipban/test_ipban.py | 6 ++---- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/deployment/terraform/resources/storage_account.tf b/deployment/terraform/resources/storage_account.tf index 66f68432..63483ea8 100644 --- a/deployment/terraform/resources/storage_account.tf +++ b/deployment/terraform/resources/storage_account.tf @@ -28,4 +28,9 @@ resource "azurerm_storage_table" "ipexceptionlist" { resource "azurerm_storage_table" "blobstoragebannedip" { name = "blobstoragebannedip" storage_account_name = azurerm_storage_account.pc.name +} + +resource "azurerm_storage_table" "testblobstoragebannedip" { + name = "testblobstoragebannedip" + storage_account_name = azurerm_storage_account.pc.name } \ No newline at end of file diff --git a/pcfuncs/ipban/__init__.py b/pcfuncs/ipban/__init__.py index bf04405a..ab513e80 100644 --- a/pcfuncs/ipban/__init__.py +++ b/pcfuncs/ipban/__init__.py @@ -7,7 +7,6 @@ from azure.monitor.query import LogsQueryClient from .constants import BANNED_IP_TABLE, STORAGE_ACCOUNT_URL - from .models import UpdateBannedIPTask diff --git a/pcfuncs/ipban/models.py b/pcfuncs/ipban/models.py index af80e788..f0709c0c 100644 --- a/pcfuncs/ipban/models.py +++ b/pcfuncs/ipban/models.py @@ -3,9 +3,7 @@ from azure.data.tables import TableClient, UpdateMode from azure.monitor.query import LogsQueryClient -from azure.monitor.query._models import ( - LogsTableRow, -) +from azure.monitor.query._models import LogsTableRow from .constants import ( LOG_ANALYTICS_WORKSPACE_ID, diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index 33c2bc50..e45fc691 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -1,14 +1,12 @@ from typing import Any, Dict, Generator, List, Tuple from unittest.mock import MagicMock -from azure.data.tables._entity import TableEntity import pytest from azure.data.tables import TableClient, TableServiceClient +from azure.data.tables._entity import TableEntity from azure.identity import DefaultAzureCredential from azure.monitor.query import LogsQueryClient -from azure.monitor.query._models import ( - LogsTableRow, -) +from azure.monitor.query._models import LogsTableRow from ipban.constants import ( STORAGE_ACCOUNT_URL, THRESHOLD_READ_COUNT_IN_GB, From ea9bcf3c6b599df29c56137be924c2497f11abca Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 10 Jun 2024 21:43:03 -0700 Subject: [PATCH 09/41] test --- deployment/terraform/resources/storage_account.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/deployment/terraform/resources/storage_account.tf b/deployment/terraform/resources/storage_account.tf index 63483ea8..66f68432 100644 --- a/deployment/terraform/resources/storage_account.tf +++ b/deployment/terraform/resources/storage_account.tf @@ -28,9 +28,4 @@ resource "azurerm_storage_table" "ipexceptionlist" { resource "azurerm_storage_table" "blobstoragebannedip" { name = "blobstoragebannedip" storage_account_name = azurerm_storage_account.pc.name -} - -resource "azurerm_storage_table" "testblobstoragebannedip" { - name = "testblobstoragebannedip" - storage_account_name = azurerm_storage_account.pc.name } \ No newline at end of file From 32f64583152765fdb56a29e2f1b39ca858e73f92 Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 10 Jun 2024 21:45:22 -0700 Subject: [PATCH 10/41] remove readme --- pcfuncs/ipban/readme.md | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 pcfuncs/ipban/readme.md diff --git a/pcfuncs/ipban/readme.md b/pcfuncs/ipban/readme.md deleted file mode 100644 index e8b7e887..00000000 --- a/pcfuncs/ipban/readme.md +++ /dev/null @@ -1,11 +0,0 @@ -# TimerTrigger - Python - -The `TimerTrigger` makes it incredibly easy to have your functions executed on a schedule. This sample demonstrates a simple use case of calling your function every 5 minutes. - -## How it works - -For a `TimerTrigger` to work, you provide a schedule in the form of a [cron expression](https://en.wikipedia.org/wiki/Cron#CRON_expression)(See the link for full details). A cron expression is a string with 6 separate expressions which represent a given schedule via patterns. The pattern we use to represent every 5 minutes is `0 */5 * * * *`. This, in plain text, means: "When seconds is equal to 0, minutes is divisible by 5, for any hour, day of the month, month, day of the week, or year". - -## Learn more - - Documentation From 3f1667befad830006500be996f8b7f8429f3066e Mon Sep 17 00:00:00 2001 From: elay Date: Tue, 11 Jun 2024 16:00:22 -0700 Subject: [PATCH 11/41] remove redundant packages and revert python version --- pcfuncs/Dockerfile | 2 +- pcfuncs/requirements.txt | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/pcfuncs/Dockerfile b/pcfuncs/Dockerfile index 888f90ce..323859d3 100644 --- a/pcfuncs/Dockerfile +++ b/pcfuncs/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/azure-functions/python:4-python3.10 +FROM mcr.microsoft.com/azure-functions/python:4-python3.8 # git required for pip installs from git RUN apt update && apt install -y git diff --git a/pcfuncs/requirements.txt b/pcfuncs/requirements.txt index dc645a9f..3483db1e 100644 --- a/pcfuncs/requirements.txt +++ b/pcfuncs/requirements.txt @@ -14,12 +14,7 @@ pillow==10.3.0 pyproj==3.3.1 pydantic>=1.9,<2.0.0 rasterio==1.3.* -azure-core==1.30.1 -azure-data-tables==12.4.0 -azure-identity==1.7.1 azure-monitor-query==1.3.0 -azure-storage-blob==12.20.0 -opencensus-ext-azure==1.0.8 pytest-mock==3.14.0 # Deployment needs to copy the local code into # the app code directory, so requires a separate From fe6bbcc44a613fd4c2786b9a2fa04cc7ffb40b2f Mon Sep 17 00:00:00 2001 From: elay Date: Tue, 11 Jun 2024 16:02:17 -0700 Subject: [PATCH 12/41] revert python version --- deployment/terraform/resources/functions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/terraform/resources/functions.tf b/deployment/terraform/resources/functions.tf index f38ce3cb..cdcf6ebf 100644 --- a/deployment/terraform/resources/functions.tf +++ b/deployment/terraform/resources/functions.tf @@ -46,7 +46,7 @@ resource "azurerm_function_app" "pcfuncs" { os_type = "linux" version = "~4" site_config { - linux_fx_version = "PYTHON|3.10" + linux_fx_version = "PYTHON|3.8" use_32_bit_worker_process = false ftps_state = "Disabled" From f3d56e17e4161fea992c6878cf2c48c6949bf039 Mon Sep 17 00:00:00 2001 From: elay Date: Tue, 11 Jun 2024 22:43:55 -0700 Subject: [PATCH 13/41] remove redundant type hints --- pcfuncs/ipban/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pcfuncs/ipban/models.py b/pcfuncs/ipban/models.py index f0709c0c..23d7ad25 100644 --- a/pcfuncs/ipban/models.py +++ b/pcfuncs/ipban/models.py @@ -18,8 +18,8 @@ def __init__( logs_query_client: LogsQueryClient, table_client: TableClient, ) -> None: - self.log_query_client: LogsQueryClient = logs_query_client - self.table_client: TableClient = table_client + self.log_query_client = logs_query_client + self.table_client = table_client def run(self) -> List[LogsTableRow]: query_result: List[LogsTableRow] = self.get_blob_logs_query_result() From f3cd2ba34c96994e51b83a5b219c6037e9dcd3a4 Mon Sep 17 00:00:00 2001 From: elay Date: Tue, 11 Jun 2024 23:20:16 -0700 Subject: [PATCH 14/41] use settings class inherited from baseSettings --- pcfuncs/ipban/__init__.py | 6 +++--- pcfuncs/ipban/config.py | 19 +++++++++++++++++++ pcfuncs/ipban/constants.py | 10 ---------- pcfuncs/ipban/models.py | 16 ++++++---------- pcfuncs/tests/ipban/test_ipban.py | 28 ++++++++++++---------------- 5 files changed, 40 insertions(+), 39 deletions(-) create mode 100644 pcfuncs/ipban/config.py delete mode 100644 pcfuncs/ipban/constants.py diff --git a/pcfuncs/ipban/__init__.py b/pcfuncs/ipban/__init__.py index ab513e80..e26b9595 100644 --- a/pcfuncs/ipban/__init__.py +++ b/pcfuncs/ipban/__init__.py @@ -6,7 +6,7 @@ from azure.identity import DefaultAzureCredential from azure.monitor.query import LogsQueryClient -from .constants import BANNED_IP_TABLE, STORAGE_ACCOUNT_URL +from .config import settings from .models import UpdateBannedIPTask @@ -18,10 +18,10 @@ def main(mytimer: func.TimerRequest) -> None: credential: DefaultAzureCredential = DefaultAzureCredential() logs_query_client: LogsQueryClient = LogsQueryClient(credential) table_service_client: TableServiceClient = TableServiceClient( - endpoint=STORAGE_ACCOUNT_URL, credential=credential + endpoint=settings.storage_account_url, credential=credential ) table_client: TableClient = table_service_client.create_table_if_not_exists( - BANNED_IP_TABLE + settings.banned_ip_table ) task: UpdateBannedIPTask = UpdateBannedIPTask(logs_query_client, table_client) task.run() diff --git a/pcfuncs/ipban/config.py b/pcfuncs/ipban/config.py new file mode 100644 index 00000000..1fe518ee --- /dev/null +++ b/pcfuncs/ipban/config.py @@ -0,0 +1,19 @@ +# config.py +from pydantic import BaseSettings + + +class Settings(BaseSettings): + # Constants related to Azure Table Storage + storage_account_url: str = "https://pctapisstagingsa.table.core.windows.net/" + banned_ip_table: str = "blobstoragebannedip" + + # Log Analytics Workspace: pc-api-loganalytics + log_analytics_workspace_id: str = "78d48390-b6bb-49a9-b7fd-a86f6522e9c4" + + # Time and threshold settings + time_window_in_hours: int = 24 + threshold_read_count_in_gb: int = 5120 + + +# Create a global settings instance +settings = Settings() diff --git a/pcfuncs/ipban/constants.py b/pcfuncs/ipban/constants.py deleted file mode 100644 index 812fd365..00000000 --- a/pcfuncs/ipban/constants.py +++ /dev/null @@ -1,10 +0,0 @@ -# Constants related to Azure Table Storage -STORAGE_ACCOUNT_URL = "https://pcfilestest.table.core.windows.net/" -BANNED_IP_TABLE = "blobstoragebannedip" - -# Log Analytics Workspace: pc-api-loganalytics -LOG_ANALYTICS_WORKSPACE_ID = "78d48390-b6bb-49a9-b7fd-a86f6522e9c4" - -# Time and threshold settings -TIME_WINDOW_IN_HOURS = 24 -THRESHOLD_READ_COUNT_IN_GB = 100 diff --git a/pcfuncs/ipban/models.py b/pcfuncs/ipban/models.py index 23d7ad25..b1b3d417 100644 --- a/pcfuncs/ipban/models.py +++ b/pcfuncs/ipban/models.py @@ -5,11 +5,7 @@ from azure.monitor.query import LogsQueryClient from azure.monitor.query._models import LogsTableRow -from .constants import ( - LOG_ANALYTICS_WORKSPACE_ID, - THRESHOLD_READ_COUNT_IN_GB, - TIME_WINDOW_IN_HOURS, -) +from .config import settings class UpdateBannedIPTask: @@ -29,14 +25,14 @@ def run(self) -> List[LogsTableRow]: def get_blob_logs_query_result(self) -> List[LogsTableRow]: query: str = f""" StorageBlobLogs - | where TimeGenerated > ago({TIME_WINDOW_IN_HOURS}h) + | where TimeGenerated > ago({settings.time_window_in_hours}h) | extend IpAddress = tostring(split(CallerIpAddress, ":")[0]) | summarize readcount = sum(ResponseBodySize) / (1024 * 1024 * 1024) by IpAddress - | where readcount > {THRESHOLD_READ_COUNT_IN_GB} + | where readcount > {settings.threshold_read_count_in_gb} """ response: Any = self.log_query_client.query_workspace( - LOG_ANALYTICS_WORKSPACE_ID, query, timespan=None + settings.log_analytics_workspace_id, query, timespan=None ) return response.tables[0].rows @@ -53,8 +49,8 @@ def update_banned_ips(self, query_result: List[LogsTableRow]) -> None: "PartitionKey": ip_address, "RowKey": ip_address, "ReadCount": read_count, - "Threshold": THRESHOLD_READ_COUNT_IN_GB, - "TimeWindow": TIME_WINDOW_IN_HOURS, + "Threshold": settings.threshold_read_count_in_gb, + "TimeWindow": settings.time_window_in_hours, } if ip_address in existing_ips: diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index e45fc691..8d41c614 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -7,11 +7,7 @@ from azure.identity import DefaultAzureCredential from azure.monitor.query import LogsQueryClient from azure.monitor.query._models import LogsTableRow -from ipban.constants import ( - STORAGE_ACCOUNT_URL, - THRESHOLD_READ_COUNT_IN_GB, - TIME_WINDOW_IN_HOURS, -) +from ipban.config import settings from ipban.models import UpdateBannedIPTask from pytest_mock import MockerFixture @@ -26,22 +22,22 @@ def populate_banned_ip_table(table_client: TableClient) -> List[Dict[str, Any]]: "PartitionKey": "192.168.1.1", "RowKey": "192.168.1.1", "ReadCount": 647, - "Threshold": THRESHOLD_READ_COUNT_IN_GB, - "TimeWindow": TIME_WINDOW_IN_HOURS, + "Threshold": settings.threshold_read_count_in_gb, + "TimeWindow": settings.time_window_in_hours, }, { "PartitionKey": "192.168.1.2", "RowKey": "192.168.1.2", "ReadCount": 214, - "Threshold": THRESHOLD_READ_COUNT_IN_GB, - "TimeWindow": TIME_WINDOW_IN_HOURS, + "Threshold": settings.threshold_read_count_in_gb, + "TimeWindow": settings.time_window_in_hours, }, { "PartitionKey": "192.168.1.3", "RowKey": "192.168.1.3", "ReadCount": 550, - "Threshold": THRESHOLD_READ_COUNT_IN_GB, - "TimeWindow": TIME_WINDOW_IN_HOURS, + "Threshold": settings.threshold_read_count_in_gb, + "TimeWindow": settings.time_window_in_hours, }, ] for entity in entities: @@ -96,7 +92,7 @@ def integration_clients( credential: DefaultAzureCredential = DefaultAzureCredential() logs_query_client: LogsQueryClient = LogsQueryClient(credential) table_service_client: TableServiceClient = TableServiceClient( - endpoint=STORAGE_ACCOUNT_URL, credential=credential + endpoint=settings.storage_account_url, credential=credential ) table_client: TableClient = table_service_client.create_table_if_not_exists( TEST_BANNED_IP_TABLE @@ -122,8 +118,8 @@ def test_update_banned_ip_integration( for ip, expected_read_count in logs_query_result: entity: TableEntity = table_client.get_entity(ip, ip) assert entity["ReadCount"] == expected_read_count - assert entity["Threshold"] == THRESHOLD_READ_COUNT_IN_GB - assert entity["TimeWindow"] == TIME_WINDOW_IN_HOURS + assert entity["Threshold"] == settings.threshold_read_count_in_gb + assert entity["TimeWindow"] == settings.time_window_in_hours def test_update_banned_ip(mock_clients: Tuple[MagicMock, TableClient]) -> None: @@ -136,5 +132,5 @@ def test_update_banned_ip(mock_clients: Tuple[MagicMock, TableClient]) -> None: for ip, expected_read_count in MOCK_LOGS_QUERY_RESULT: entity = table_client.get_entity(ip, ip) assert entity["ReadCount"] == expected_read_count - assert entity["Threshold"] == THRESHOLD_READ_COUNT_IN_GB - assert entity["TimeWindow"] == TIME_WINDOW_IN_HOURS + assert entity["Threshold"] == settings.threshold_read_count_in_gb + assert entity["TimeWindow"] == settings.time_window_in_hours From 4e9021a8fff67a2e8593b65ff891f2a17e020f81 Mon Sep 17 00:00:00 2001 From: elay Date: Wed, 12 Jun 2024 00:09:00 -0700 Subject: [PATCH 15/41] change kql --- pcfuncs/ipban/models.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pcfuncs/ipban/models.py b/pcfuncs/ipban/models.py index b1b3d417..7939d2fe 100644 --- a/pcfuncs/ipban/models.py +++ b/pcfuncs/ipban/models.py @@ -24,12 +24,14 @@ def run(self) -> List[LogsTableRow]: def get_blob_logs_query_result(self) -> List[LogsTableRow]: query: str = f""" - StorageBlobLogs - | where TimeGenerated > ago({settings.time_window_in_hours}h) - | extend IpAddress = tostring(split(CallerIpAddress, ":")[0]) - | summarize readcount = sum(ResponseBodySize) / (1024 * 1024 * 1024) - by IpAddress - | where readcount > {settings.threshold_read_count_in_gb} + StorageBlobLogs + | where TimeGenerated > ago({settings.time_window_in_hours}h) + | extend IpAddress = tostring(split(CallerIpAddress, ":")[0]) + | where OperationName == 'GetBlob' + | where not(ipv4_is_private(IpAddress)) + | summarize readcount = sum(ResponseBodySize) / (1024 * 1024 * 1024) + by IpAddress + | where readcount > {settings.threshold_read_count_in_gb} """ response: Any = self.log_query_client.query_workspace( settings.log_analytics_workspace_id, query, timespan=None From 1d9f5f671a40b015f9fc8871740eb5ec821fb2b3 Mon Sep 17 00:00:00 2001 From: elay Date: Wed, 12 Jun 2024 08:43:11 -0700 Subject: [PATCH 16/41] use placeholder for timer schedule --- pcfuncs/ipban/function.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcfuncs/ipban/function.json b/pcfuncs/ipban/function.json index 5f8080ed..d2156a61 100644 --- a/pcfuncs/ipban/function.json +++ b/pcfuncs/ipban/function.json @@ -5,7 +5,7 @@ "name": "mytimer", "type": "timerTrigger", "direction": "in", - "schedule": "0 0 * * * *" + "schedule": "%TIMER_SCHEDULE%" } ] } \ No newline at end of file From 5ade5421f571cb190a076d440dcf6a669b0613a0 Mon Sep 17 00:00:00 2001 From: elay Date: Wed, 12 Jun 2024 10:43:02 -0700 Subject: [PATCH 17/41] change test parameter --- pcfuncs/tests/ipban/test_ipban.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index 8d41c614..c27d28a3 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -11,7 +11,7 @@ from ipban.models import UpdateBannedIPTask from pytest_mock import MockerFixture -MOCK_LOGS_QUERY_RESULT = [("192.168.1.1", 170), ("192.168.1.4", 420)] +MOCK_LOGS_QUERY_RESULT = [("192.168.1.1", 8000), ("192.168.1.4", 12000)] TEST_BANNED_IP_TABLE = "testblobstoragebannedip" From fa70e5eb7e2b4b3ba782b1e5bfe879d5d558abd5 Mon Sep 17 00:00:00 2001 From: elay Date: Wed, 12 Jun 2024 11:13:31 -0700 Subject: [PATCH 18/41] update assertions and logger --- pcfuncs/ipban/__init__.py | 4 +- pcfuncs/tests/ipban/test_ipban.py | 101 ++++++++++++++---------------- 2 files changed, 50 insertions(+), 55 deletions(-) diff --git a/pcfuncs/ipban/__init__.py b/pcfuncs/ipban/__init__.py index e26b9595..6e3d66fd 100644 --- a/pcfuncs/ipban/__init__.py +++ b/pcfuncs/ipban/__init__.py @@ -9,12 +9,14 @@ from .config import settings from .models import UpdateBannedIPTask +logger = logging.getLogger(__name__) + def main(mytimer: func.TimerRequest) -> None: utc_timestamp: str = ( datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat() ) - logging.info("Updating the ip ban list at %s", utc_timestamp) + logger.info("Updating the ip ban list at %s", utc_timestamp) credential: DefaultAzureCredential = DefaultAzureCredential() logs_query_client: LogsQueryClient = LogsQueryClient(credential) table_service_client: TableServiceClient = TableServiceClient( diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index c27d28a3..d2a17664 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -1,3 +1,5 @@ +import logging +import uuid from typing import Any, Dict, Generator, List, Tuple from unittest.mock import MagicMock @@ -12,47 +14,38 @@ from pytest_mock import MockerFixture MOCK_LOGS_QUERY_RESULT = [("192.168.1.1", 8000), ("192.168.1.4", 12000)] -TEST_BANNED_IP_TABLE = "testblobstoragebannedip" - - -def populate_banned_ip_table(table_client: TableClient) -> List[Dict[str, Any]]: - print("Populating the table") - entities: List[Dict[str, Any]] = [ - { - "PartitionKey": "192.168.1.1", - "RowKey": "192.168.1.1", - "ReadCount": 647, - "Threshold": settings.threshold_read_count_in_gb, - "TimeWindow": settings.time_window_in_hours, - }, - { - "PartitionKey": "192.168.1.2", - "RowKey": "192.168.1.2", - "ReadCount": 214, - "Threshold": settings.threshold_read_count_in_gb, - "TimeWindow": settings.time_window_in_hours, - }, - { - "PartitionKey": "192.168.1.3", - "RowKey": "192.168.1.3", - "ReadCount": 550, - "Threshold": settings.threshold_read_count_in_gb, - "TimeWindow": settings.time_window_in_hours, - }, - ] - for entity in entities: +TEST_ID = uuid.uuid4() +TEST_BANNED_IP_TABLE = f"testblobstoragebannedip-{TEST_ID}" + +logger = logging.getLogger(__name__) +PREPOPULATED_ENTITIES = [ + { + "PartitionKey": "192.168.1.1", + "RowKey": "192.168.1.1", + "ReadCount": 647, + "Threshold": settings.threshold_read_count_in_gb, + "TimeWindow": settings.time_window_in_hours, + }, + { + "PartitionKey": "192.168.1.2", + "RowKey": "192.168.1.2", + "ReadCount": 214, + "Threshold": settings.threshold_read_count_in_gb, + "TimeWindow": settings.time_window_in_hours, + }, + { + "PartitionKey": "192.168.1.3", + "RowKey": "192.168.1.3", + "ReadCount": 550, + "Threshold": settings.threshold_read_count_in_gb, + "TimeWindow": settings.time_window_in_hours, + }, +] + + +def populate_banned_ip_table(table_client: TableClient) -> None: + for entity in PREPOPULATED_ENTITIES: table_client.create_entity(entity) - return entities - - -def clear_table(table_client: TableClient) -> None: - entities = list(table_client.list_entities()) - for entity in entities: - table_client.delete_entity( - partition_key=entity["PartitionKey"], row_key=entity["RowKey"] - ) - entities = list(table_client.list_entities()) - assert len(entities) == 0 @pytest.fixture @@ -70,19 +63,18 @@ def mock_clients( "TableEndpoint=http://azurite:10002/devstoreaccount1;" ) # Use Azurite for unit tests and populate the table with initial data - table_service: TableServiceClient = TableServiceClient.from_connection_string( - CONNECTION_STRING + table_service_client: TableServiceClient = ( + TableServiceClient.from_connection_string(CONNECTION_STRING) ) - table_client: TableClient = table_service.create_table_if_not_exists( + table_client: TableClient = table_service_client.create_table_if_not_exists( table_name=TEST_BANNED_IP_TABLE ) - # Pre-populate the banned ip table populate_banned_ip_table(table_client) yield logs_query_client, table_client - # Clear all entities from the table - clear_table(table_client) + # Delete the test table + table_service_client.delete_table(TEST_BANNED_IP_TABLE) @pytest.fixture @@ -100,21 +92,22 @@ def integration_clients( # Pre-populate the banned ip table populate_banned_ip_table(table_client) yield logs_query_client, table_client - # Clear all entities from the table - clear_table(table_client) + + # Delete the test table + table_service_client.delete_table(TEST_BANNED_IP_TABLE) @pytest.mark.integration def test_update_banned_ip_integration( integration_clients: Tuple[LogsQueryClient, TableClient] ) -> None: - print("Integration test is running") + logger.info(f"Test id: {TEST_ID} - integration test is running") logs_query_client, table_client = integration_clients + assert len(list(table_client.list_entities())) == len(PREPOPULATED_ENTITIES) task: UpdateBannedIPTask = UpdateBannedIPTask(logs_query_client, table_client) # retrieve the logs query result from pc-api-loganalytics logs_query_result: List[LogsTableRow] = task.run() - entities = list(table_client.list_entities()) - assert len(logs_query_result) == len(entities) + assert len(list(table_client.list_entities())) == len(logs_query_result) for ip, expected_read_count in logs_query_result: entity: TableEntity = table_client.get_entity(ip, ip) assert entity["ReadCount"] == expected_read_count @@ -123,12 +116,12 @@ def test_update_banned_ip_integration( def test_update_banned_ip(mock_clients: Tuple[MagicMock, TableClient]) -> None: - print("Unit test is running") + logger.info(f"Test id: {TEST_ID} - unit test is running") mock_logs_query_client, table_client = mock_clients + assert len(list(table_client.list_entities())) == len(PREPOPULATED_ENTITIES) task: UpdateBannedIPTask = UpdateBannedIPTask(mock_logs_query_client, table_client) task.run() - entities = list(table_client.list_entities()) - assert len(entities) == len(MOCK_LOGS_QUERY_RESULT) + assert len(list(table_client.list_entities())) == len(MOCK_LOGS_QUERY_RESULT) for ip, expected_read_count in MOCK_LOGS_QUERY_RESULT: entity = table_client.get_entity(ip, ip) assert entity["ReadCount"] == expected_read_count From b115435ed7628e0cca99cc353772a4834a4ec5d1 Mon Sep 17 00:00:00 2001 From: elay Date: Wed, 12 Jun 2024 11:20:26 -0700 Subject: [PATCH 19/41] remove import --- pcfuncs/tests/ipban/test_ipban.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index d2a17664..1dda43b4 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -1,6 +1,6 @@ import logging import uuid -from typing import Any, Dict, Generator, List, Tuple +from typing import Any, Generator, List, Tuple from unittest.mock import MagicMock import pytest From e6cce58165ca65c6d86d93e9a36e5048a991ae58 Mon Sep 17 00:00:00 2001 From: elay Date: Wed, 12 Jun 2024 11:22:26 -0700 Subject: [PATCH 20/41] remove dash in table name --- pcfuncs/tests/ipban/test_ipban.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index 1dda43b4..d9757512 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -15,7 +15,7 @@ MOCK_LOGS_QUERY_RESULT = [("192.168.1.1", 8000), ("192.168.1.4", 12000)] TEST_ID = uuid.uuid4() -TEST_BANNED_IP_TABLE = f"testblobstoragebannedip-{TEST_ID}" +TEST_BANNED_IP_TABLE = f"testblobstoragebannedip{TEST_ID}" logger = logging.getLogger(__name__) PREPOPULATED_ENTITIES = [ From 21b4010076be9d2b35e5d575c561bf9a96a72de3 Mon Sep 17 00:00:00 2001 From: elay Date: Wed, 12 Jun 2024 11:42:11 -0700 Subject: [PATCH 21/41] update test id --- pcfuncs/tests/ipban/test_ipban.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index d9757512..5716dcea 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -14,7 +14,7 @@ from pytest_mock import MockerFixture MOCK_LOGS_QUERY_RESULT = [("192.168.1.1", 8000), ("192.168.1.4", 12000)] -TEST_ID = uuid.uuid4() +TEST_ID = str(uuid.uuid4()).replace("-", "") # dash is not allowed in table name TEST_BANNED_IP_TABLE = f"testblobstoragebannedip{TEST_ID}" logger = logging.getLogger(__name__) From 342a3455093519dc0025fb2e0f3704b726674967 Mon Sep 17 00:00:00 2001 From: elay Date: Wed, 12 Jun 2024 11:46:20 -0700 Subject: [PATCH 22/41] format --- pcfuncs/tests/ipban/test_ipban.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index 5716dcea..e7f258db 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -14,7 +14,7 @@ from pytest_mock import MockerFixture MOCK_LOGS_QUERY_RESULT = [("192.168.1.1", 8000), ("192.168.1.4", 12000)] -TEST_ID = str(uuid.uuid4()).replace("-", "") # dash is not allowed in table name +TEST_ID = str(uuid.uuid4()).replace("-", "") # dash is not allowed in table name TEST_BANNED_IP_TABLE = f"testblobstoragebannedip{TEST_ID}" logger = logging.getLogger(__name__) From d2bdaccf5fc4edebe7219caefc497af7d23fef07 Mon Sep 17 00:00:00 2001 From: elay Date: Wed, 12 Jun 2024 12:15:41 -0700 Subject: [PATCH 23/41] add no-integration flag --- scripts/bin/test-funcs | 19 ++++++++++++++++++- scripts/test | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/scripts/bin/test-funcs b/scripts/bin/test-funcs index 22b1d2db..64bb06eb 100755 --- a/scripts/bin/test-funcs +++ b/scripts/bin/test-funcs @@ -15,6 +15,23 @@ This scripts is meant to be run inside the funcs container. " } +while [[ $# -gt 0 ]]; do case $1 in + --no-integration) + INTEGRATION="--no-integration" + shift + ;; + --help) + usage + exit 0 + shift + ;; + *) + usage "Unknown parameter passed: $1" + shift + shift + ;; + esac done + if [ "${BASH_SOURCE[0]}" = "${0}" ]; then echo "Running mypy for funcs..." @@ -27,6 +44,6 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then flake8 pcfuncs echo "Running unit tests for funcs..." - python -m pytest pcfuncs/tests + python -m pytest pcfuncs/tests ${INTEGRATION:+$INTEGRATION} fi diff --git a/scripts/test b/scripts/test index 35685dff..8f283525 100755 --- a/scripts/test +++ b/scripts/test @@ -94,7 +94,7 @@ if [ "${BASH_SOURCE[0]}" = "${0}" ]; then docker-compose \ -f docker-compose.yml \ run --rm \ - funcs /bin/bash -c "cd /opt/src && scripts/bin/test-funcs" + funcs /bin/bash -c "cd /opt/src && scripts/bin/test-funcs ${NO_INTEGRATION}" fi fi From d70c16600b10bbdb1d0ab6e990d35c6e4dff17c9 Mon Sep 17 00:00:00 2001 From: elay Date: Wed, 12 Jun 2024 16:10:23 -0700 Subject: [PATCH 24/41] add dependencies --- pcfuncs/requirements-deploy.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pcfuncs/requirements-deploy.txt b/pcfuncs/requirements-deploy.txt index 8673ad28..2c11104d 100644 --- a/pcfuncs/requirements-deploy.txt +++ b/pcfuncs/requirements-deploy.txt @@ -14,7 +14,8 @@ pillow==10.3.0 pyproj==3.3.1 pydantic>=1.9,<2.0.0 rasterio==1.3.* - +azure-monitor-query==1.3.0 +pytest-mock==3.14.0 # The deploy process needs symlinks to bring in # pctasks libraries. Symlink is created in deploy script ./pccommon_linked From 7334fea0c77764c5f1d6eca9b4015e990dd68cfb Mon Sep 17 00:00:00 2001 From: elay Date: Wed, 12 Jun 2024 16:11:49 -0700 Subject: [PATCH 25/41] test --- pcfuncs/ipban/function.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcfuncs/ipban/function.json b/pcfuncs/ipban/function.json index d2156a61..ef469d77 100644 --- a/pcfuncs/ipban/function.json +++ b/pcfuncs/ipban/function.json @@ -8,4 +8,4 @@ "schedule": "%TIMER_SCHEDULE%" } ] -} \ No newline at end of file +} From bc69adbf253368fa0db5442032f09bf0369b6084 Mon Sep 17 00:00:00 2001 From: elay Date: Fri, 14 Jun 2024 11:56:40 -0700 Subject: [PATCH 26/41] change trigger to run every hour --- pcfuncs/ipban/function.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcfuncs/ipban/function.json b/pcfuncs/ipban/function.json index ef469d77..2b55fa8e 100644 --- a/pcfuncs/ipban/function.json +++ b/pcfuncs/ipban/function.json @@ -5,7 +5,7 @@ "name": "mytimer", "type": "timerTrigger", "direction": "in", - "schedule": "%TIMER_SCHEDULE%" + "schedule": "0 */1 * * *" } ] } From b6aa998ec3a8a7d8d9da0088bffb9c939fcbe2ad Mon Sep 17 00:00:00 2001 From: elay Date: Thu, 20 Jun 2024 10:25:21 -0700 Subject: [PATCH 27/41] use azure clients as context manager --- pcfuncs/ipban/__init__.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pcfuncs/ipban/__init__.py b/pcfuncs/ipban/__init__.py index 6e3d66fd..7a7d1ce0 100644 --- a/pcfuncs/ipban/__init__.py +++ b/pcfuncs/ipban/__init__.py @@ -2,7 +2,7 @@ import logging import azure.functions as func -from azure.data.tables import TableClient, TableServiceClient +from azure.data.tables import TableServiceClient from azure.identity import DefaultAzureCredential from azure.monitor.query import LogsQueryClient @@ -18,12 +18,10 @@ def main(mytimer: func.TimerRequest) -> None: ) logger.info("Updating the ip ban list at %s", utc_timestamp) credential: DefaultAzureCredential = DefaultAzureCredential() - logs_query_client: LogsQueryClient = LogsQueryClient(credential) - table_service_client: TableServiceClient = TableServiceClient( + with LogsQueryClient(credential) as logs_query_client, TableServiceClient( endpoint=settings.storage_account_url, credential=credential - ) - table_client: TableClient = table_service_client.create_table_if_not_exists( + ) as table_service_client, table_service_client.create_table_if_not_exists( settings.banned_ip_table - ) - task: UpdateBannedIPTask = UpdateBannedIPTask(logs_query_client, table_client) - task.run() + ) as table_client: + task = UpdateBannedIPTask(logs_query_client, table_client) + task.run() From 354352a1edaf295c56ebb65f2d64a351b4668b1c Mon Sep 17 00:00:00 2001 From: elay Date: Thu, 20 Jun 2024 15:39:24 -0700 Subject: [PATCH 28/41] add context managers in test --- pcfuncs/ipban/models.py | 2 +- pcfuncs/tests/ipban/test_ipban.py | 38 +++++++++++++++---------------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/pcfuncs/ipban/models.py b/pcfuncs/ipban/models.py index 7939d2fe..b59aef03 100644 --- a/pcfuncs/ipban/models.py +++ b/pcfuncs/ipban/models.py @@ -40,7 +40,7 @@ def get_blob_logs_query_result(self) -> List[LogsTableRow]: def update_banned_ips(self, query_result: List[LogsTableRow]) -> None: existing_ips = { - entity["RowKey"]: entity for entity in self.table_client.list_entities() + entity["RowKey"] for entity in self.table_client.list_entities() } result_ips: Set[str] = set() for result in query_result: diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index e7f258db..52b46e17 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -63,18 +63,17 @@ def mock_clients( "TableEndpoint=http://azurite:10002/devstoreaccount1;" ) # Use Azurite for unit tests and populate the table with initial data - table_service_client: TableServiceClient = ( - TableServiceClient.from_connection_string(CONNECTION_STRING) - ) - table_client: TableClient = table_service_client.create_table_if_not_exists( + with TableServiceClient.from_connection_string( + CONNECTION_STRING + ) as table_service_client, table_service_client.create_table_if_not_exists( table_name=TEST_BANNED_IP_TABLE - ) - # Pre-populate the banned ip table - populate_banned_ip_table(table_client) - yield logs_query_client, table_client + ) as table_client: + # Pre-populate the banned ip table + populate_banned_ip_table(table_client) + yield logs_query_client, table_client - # Delete the test table - table_service_client.delete_table(TEST_BANNED_IP_TABLE) + # Delete the test table + table_service_client.delete_table(TEST_BANNED_IP_TABLE) @pytest.fixture @@ -82,19 +81,18 @@ def integration_clients( mocker: MockerFixture, ) -> Generator[Tuple[LogsQueryClient, TableClient], Any, None]: credential: DefaultAzureCredential = DefaultAzureCredential() - logs_query_client: LogsQueryClient = LogsQueryClient(credential) - table_service_client: TableServiceClient = TableServiceClient( + with LogsQueryClient(credential) as logs_query_client, TableServiceClient( endpoint=settings.storage_account_url, credential=credential - ) - table_client: TableClient = table_service_client.create_table_if_not_exists( + ) as table_service_client, table_service_client.create_table_if_not_exists( TEST_BANNED_IP_TABLE - ) - # Pre-populate the banned ip table - populate_banned_ip_table(table_client) - yield logs_query_client, table_client + ) as table_client: + + # Pre-populate the banned ip table + populate_banned_ip_table(table_client) + yield logs_query_client, table_client - # Delete the test table - table_service_client.delete_table(TEST_BANNED_IP_TABLE) + # Delete the test table + table_service_client.delete_table(TEST_BANNED_IP_TABLE) @pytest.mark.integration From e95abd5484d437db7b2a79ef594dbc54cb817df7 Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 24 Jun 2024 14:05:19 -0700 Subject: [PATCH 29/41] role assignment for function app --- deployment/terraform/resources/functions.tf | 26 +++++++++++++++++++++ deployment/terraform/resources/providers.tf | 6 +++++ deployment/terraform/resources/variables.tf | 9 +++++++ deployment/terraform/staging/main.tf | 1 + 4 files changed, 42 insertions(+) diff --git a/deployment/terraform/resources/functions.tf b/deployment/terraform/resources/functions.tf index e77b3082..8b144d7e 100644 --- a/deployment/terraform/resources/functions.tf +++ b/deployment/terraform/resources/functions.tf @@ -77,3 +77,29 @@ resource "azurerm_role_assignment" "function-app-animation-container-access" { azurerm_function_app.pcfuncs ] } + +resource "azurerm_role_assignment" "function-app-storage-table-data-contributor" { + scope = azurerm_storage_account.pc.id + role_definition_name = "Storage Table Data Contributor" + principal_id = azurerm_function_app.pcfuncs.identity[0].principal_id + + depends_on = [ + azurerm_function_app.pcfuncs + ] +} + +data "azurerm_log_analytics_workspace" "log_analytics_workspace" { + provider = azurerm.log_analytics + name = var.log_analytics_workspace_name + resource_group_name = var.pc_resources_rg +} + +resource "azurerm_role_assignment" "function-app-log-analytics-access" { + scope = data.azurerm_log_analytics_workspace.log_analytics_workspace.id + role_definition_name = "Log Analytics Contributor" + principal_id = azurerm_function_app.pcfuncs.identity[0].principal_id + + depends_on = [ + azurerm_function_app.pcfuncs + ] +} \ No newline at end of file diff --git a/deployment/terraform/resources/providers.tf b/deployment/terraform/resources/providers.tf index 2404234d..915deba8 100644 --- a/deployment/terraform/resources/providers.tf +++ b/deployment/terraform/resources/providers.tf @@ -8,6 +8,12 @@ provider "azurerm" { # storage_use_azuread = true } +provider "azurerm" { + alias = "log_analytics" + subscription_id = "9da7523a-cb61-4c3e-b1d4-afa5fc6d2da9" + features {} +} + terraform { required_version = ">= 0.13" diff --git a/deployment/terraform/resources/variables.tf b/deployment/terraform/resources/variables.tf index c760a708..c5323521 100644 --- a/deployment/terraform/resources/variables.tf +++ b/deployment/terraform/resources/variables.tf @@ -11,6 +11,11 @@ variable "pc_test_resources_rg" { default = "pc-test-manual-resources" } +variable "pc_resources_rg" { + type = string + default = "pc-manual-resources" +} + variable "pc_test_resources_kv" { type = string default = "pc-test-deploy-secrets" @@ -123,6 +128,10 @@ variable "image_output_storage_url" { type = string } +variable "log_analytics_workspace_name" { + type = string +} + # ----------------- # Local variables diff --git a/deployment/terraform/staging/main.tf b/deployment/terraform/staging/main.tf index 75567650..997ddc7f 100644 --- a/deployment/terraform/staging/main.tf +++ b/deployment/terraform/staging/main.tf @@ -22,6 +22,7 @@ module "resources" { animation_output_storage_url = "https://pcfilestest.blob.core.windows.net/output/animations" image_output_storage_url = "https://pcfilestest.blob.core.windows.net/output/images" + log_analytics_workspace_name = "pc-api-loganalytics" } terraform { From 1753331de7c1e3721a9fa6c91f4243c7b0cd362f Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 24 Jun 2024 15:33:45 -0700 Subject: [PATCH 30/41] change LAW name --- deployment/terraform/resources/functions.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/terraform/resources/functions.tf b/deployment/terraform/resources/functions.tf index 8b144d7e..e42cfc1f 100644 --- a/deployment/terraform/resources/functions.tf +++ b/deployment/terraform/resources/functions.tf @@ -88,14 +88,14 @@ resource "azurerm_role_assignment" "function-app-storage-table-data-contributor" ] } -data "azurerm_log_analytics_workspace" "log_analytics_workspace" { +data "azurerm_log_analytics_workspace" "prod_log_analytics_workspace" { provider = azurerm.log_analytics name = var.log_analytics_workspace_name resource_group_name = var.pc_resources_rg } resource "azurerm_role_assignment" "function-app-log-analytics-access" { - scope = data.azurerm_log_analytics_workspace.log_analytics_workspace.id + scope = data.azurerm_log_analytics_workspace.prod_log_analytics_workspace.id role_definition_name = "Log Analytics Contributor" principal_id = azurerm_function_app.pcfuncs.identity[0].principal_id From 47e5790163ab1847400521bbb187f4dcbf5f3027 Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 24 Jun 2024 15:37:10 -0700 Subject: [PATCH 31/41] change role --- deployment/terraform/resources/functions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/terraform/resources/functions.tf b/deployment/terraform/resources/functions.tf index e42cfc1f..6ca80094 100644 --- a/deployment/terraform/resources/functions.tf +++ b/deployment/terraform/resources/functions.tf @@ -96,7 +96,7 @@ data "azurerm_log_analytics_workspace" "prod_log_analytics_workspace" { resource "azurerm_role_assignment" "function-app-log-analytics-access" { scope = data.azurerm_log_analytics_workspace.prod_log_analytics_workspace.id - role_definition_name = "Log Analytics Contributor" + role_definition_name = "Log Analytics Reader" principal_id = azurerm_function_app.pcfuncs.identity[0].principal_id depends_on = [ From c6c012bd698a12b5d0ea6cd0287ecad20d5e5f5c Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 24 Jun 2024 16:05:06 -0700 Subject: [PATCH 32/41] change provider's name --- deployment/terraform/resources/functions.tf | 2 +- deployment/terraform/resources/providers.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/terraform/resources/functions.tf b/deployment/terraform/resources/functions.tf index 6ca80094..e9ea7004 100644 --- a/deployment/terraform/resources/functions.tf +++ b/deployment/terraform/resources/functions.tf @@ -89,7 +89,7 @@ resource "azurerm_role_assignment" "function-app-storage-table-data-contributor" } data "azurerm_log_analytics_workspace" "prod_log_analytics_workspace" { - provider = azurerm.log_analytics + provider = azurerm.planetary_computer_subscription name = var.log_analytics_workspace_name resource_group_name = var.pc_resources_rg } diff --git a/deployment/terraform/resources/providers.tf b/deployment/terraform/resources/providers.tf index 915deba8..d71323b0 100644 --- a/deployment/terraform/resources/providers.tf +++ b/deployment/terraform/resources/providers.tf @@ -9,7 +9,7 @@ provider "azurerm" { } provider "azurerm" { - alias = "log_analytics" + alias = "planetary_computer_subscription" subscription_id = "9da7523a-cb61-4c3e-b1d4-afa5fc6d2da9" features {} } From e4715fe2d4b67f4669cf6cf36a487498c2b89f86 Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 24 Jun 2024 16:16:12 -0700 Subject: [PATCH 33/41] change name of LAW --- deployment/terraform/resources/functions.tf | 2 +- deployment/terraform/resources/variables.tf | 2 +- deployment/terraform/staging/main.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/deployment/terraform/resources/functions.tf b/deployment/terraform/resources/functions.tf index e9ea7004..fb1579db 100644 --- a/deployment/terraform/resources/functions.tf +++ b/deployment/terraform/resources/functions.tf @@ -90,7 +90,7 @@ resource "azurerm_role_assignment" "function-app-storage-table-data-contributor" data "azurerm_log_analytics_workspace" "prod_log_analytics_workspace" { provider = azurerm.planetary_computer_subscription - name = var.log_analytics_workspace_name + name = var.prod_log_analytics_workspace_name resource_group_name = var.pc_resources_rg } diff --git a/deployment/terraform/resources/variables.tf b/deployment/terraform/resources/variables.tf index c5323521..1604d6c2 100644 --- a/deployment/terraform/resources/variables.tf +++ b/deployment/terraform/resources/variables.tf @@ -128,7 +128,7 @@ variable "image_output_storage_url" { type = string } -variable "log_analytics_workspace_name" { +variable "prod_log_analytics_workspace_name" { type = string } diff --git a/deployment/terraform/staging/main.tf b/deployment/terraform/staging/main.tf index 997ddc7f..2dd7ec7e 100644 --- a/deployment/terraform/staging/main.tf +++ b/deployment/terraform/staging/main.tf @@ -22,7 +22,7 @@ module "resources" { animation_output_storage_url = "https://pcfilestest.blob.core.windows.net/output/animations" image_output_storage_url = "https://pcfilestest.blob.core.windows.net/output/images" - log_analytics_workspace_name = "pc-api-loganalytics" + prod_log_analytics_workspace_name = "pc-api-loganalytics" } terraform { From c69460afc0ba054ae3b49b42df82afe5dd790ad7 Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 24 Jun 2024 16:30:03 -0700 Subject: [PATCH 34/41] better readability --- pcfuncs/ipban/__init__.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/pcfuncs/ipban/__init__.py b/pcfuncs/ipban/__init__.py index 7a7d1ce0..0d126014 100644 --- a/pcfuncs/ipban/__init__.py +++ b/pcfuncs/ipban/__init__.py @@ -17,11 +17,9 @@ def main(mytimer: func.TimerRequest) -> None: datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat() ) logger.info("Updating the ip ban list at %s", utc_timestamp) - credential: DefaultAzureCredential = DefaultAzureCredential() - with LogsQueryClient(credential) as logs_query_client, TableServiceClient( - endpoint=settings.storage_account_url, credential=credential - ) as table_service_client, table_service_client.create_table_if_not_exists( - settings.banned_ip_table - ) as table_client: - task = UpdateBannedIPTask(logs_query_client, table_client) - task.run() + credential: DefaultAzureCredential = DefaultAzureCredential() + with LogsQueryClient(credential) as logs_query_client: + with TableServiceClient(endpoint=settings.storage_account_url, credential=credential) as table_service_client: + with table_service_client.create_table_if_not_exists(settings.banned_ip_table) as table_client: + task = UpdateBannedIPTask(logs_query_client, table_client) + task.run() From 0c1bb141a7b210c59a20795de17184b09ac99368 Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 24 Jun 2024 16:35:20 -0700 Subject: [PATCH 35/41] better readability --- pcfuncs/tests/ipban/test_ipban.py | 46 ++++++++++++++++--------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index 52b46e17..848865eb 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -62,18 +62,19 @@ def mock_clients( "Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" "TableEndpoint=http://azurite:10002/devstoreaccount1;" ) + # Use Azurite for unit tests and populate the table with initial data - with TableServiceClient.from_connection_string( - CONNECTION_STRING - ) as table_service_client, table_service_client.create_table_if_not_exists( - table_name=TEST_BANNED_IP_TABLE - ) as table_client: - # Pre-populate the banned ip table - populate_banned_ip_table(table_client) - yield logs_query_client, table_client + with TableServiceClient.from_connection_string(CONNECTION_STRING) as table_service_client: + with table_service_client.create_table_if_not_exists(table_name=TEST_BANNED_IP_TABLE) as table_client: + + # Pre-populate the banned IP table + populate_banned_ip_table(table_client) + + # Yield the clients for use + yield logs_query_client, table_client - # Delete the test table - table_service_client.delete_table(TEST_BANNED_IP_TABLE) + # Delete the test table + table_service_client.delete_table(TEST_BANNED_IP_TABLE) @pytest.fixture @@ -81,18 +82,19 @@ def integration_clients( mocker: MockerFixture, ) -> Generator[Tuple[LogsQueryClient, TableClient], Any, None]: credential: DefaultAzureCredential = DefaultAzureCredential() - with LogsQueryClient(credential) as logs_query_client, TableServiceClient( - endpoint=settings.storage_account_url, credential=credential - ) as table_service_client, table_service_client.create_table_if_not_exists( - TEST_BANNED_IP_TABLE - ) as table_client: - - # Pre-populate the banned ip table - populate_banned_ip_table(table_client) - yield logs_query_client, table_client - - # Delete the test table - table_service_client.delete_table(TEST_BANNED_IP_TABLE) + with LogsQueryClient(credential) as logs_query_client: + with TableServiceClient(endpoint=settings.storage_account_url, credential=credential) as table_service_client: + with table_service_client.create_table_if_not_exists(TEST_BANNED_IP_TABLE) as table_client: + + # Pre-populate the banned IP table + populate_banned_ip_table(table_client) + + # Yield the clients for use + yield logs_query_client, table_client + + # Delete the test table + table_service_client.delete_table(TEST_BANNED_IP_TABLE) + @pytest.mark.integration From c546d7f75c262f18ee13aba2b2f786e2c5b59397 Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 24 Jun 2024 16:41:39 -0700 Subject: [PATCH 36/41] format --- pcfuncs/ipban/__init__.py | 10 +++++++--- pcfuncs/tests/ipban/test_ipban.py | 23 +++++++++++++++-------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/pcfuncs/ipban/__init__.py b/pcfuncs/ipban/__init__.py index 0d126014..72f0e645 100644 --- a/pcfuncs/ipban/__init__.py +++ b/pcfuncs/ipban/__init__.py @@ -17,9 +17,13 @@ def main(mytimer: func.TimerRequest) -> None: datetime.datetime.utcnow().replace(tzinfo=datetime.timezone.utc).isoformat() ) logger.info("Updating the ip ban list at %s", utc_timestamp) - credential: DefaultAzureCredential = DefaultAzureCredential() + credential: DefaultAzureCredential = DefaultAzureCredential() with LogsQueryClient(credential) as logs_query_client: - with TableServiceClient(endpoint=settings.storage_account_url, credential=credential) as table_service_client: - with table_service_client.create_table_if_not_exists(settings.banned_ip_table) as table_client: + with TableServiceClient( + endpoint=settings.storage_account_url, credential=credential + ) as table_service_client: + with table_service_client.create_table_if_not_exists( + settings.banned_ip_table + ) as table_client: task = UpdateBannedIPTask(logs_query_client, table_client) task.run() diff --git a/pcfuncs/tests/ipban/test_ipban.py b/pcfuncs/tests/ipban/test_ipban.py index 848865eb..749c4fd7 100644 --- a/pcfuncs/tests/ipban/test_ipban.py +++ b/pcfuncs/tests/ipban/test_ipban.py @@ -64,12 +64,16 @@ def mock_clients( ) # Use Azurite for unit tests and populate the table with initial data - with TableServiceClient.from_connection_string(CONNECTION_STRING) as table_service_client: - with table_service_client.create_table_if_not_exists(table_name=TEST_BANNED_IP_TABLE) as table_client: - + with TableServiceClient.from_connection_string( + CONNECTION_STRING + ) as table_service_client: + with table_service_client.create_table_if_not_exists( + table_name=TEST_BANNED_IP_TABLE + ) as table_client: + # Pre-populate the banned IP table populate_banned_ip_table(table_client) - + # Yield the clients for use yield logs_query_client, table_client @@ -83,12 +87,16 @@ def integration_clients( ) -> Generator[Tuple[LogsQueryClient, TableClient], Any, None]: credential: DefaultAzureCredential = DefaultAzureCredential() with LogsQueryClient(credential) as logs_query_client: - with TableServiceClient(endpoint=settings.storage_account_url, credential=credential) as table_service_client: - with table_service_client.create_table_if_not_exists(TEST_BANNED_IP_TABLE) as table_client: + with TableServiceClient( + endpoint=settings.storage_account_url, credential=credential + ) as table_service_client: + with table_service_client.create_table_if_not_exists( + TEST_BANNED_IP_TABLE + ) as table_client: # Pre-populate the banned IP table populate_banned_ip_table(table_client) - + # Yield the clients for use yield logs_query_client, table_client @@ -96,7 +104,6 @@ def integration_clients( table_service_client.delete_table(TEST_BANNED_IP_TABLE) - @pytest.mark.integration def test_update_banned_ip_integration( integration_clients: Tuple[LogsQueryClient, TableClient] From d941b496114c86e0f13791c3a4b53123c255907b Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 24 Jun 2024 16:51:15 -0700 Subject: [PATCH 37/41] add logging --- pcfuncs/ipban/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pcfuncs/ipban/models.py b/pcfuncs/ipban/models.py index b59aef03..71ec5021 100644 --- a/pcfuncs/ipban/models.py +++ b/pcfuncs/ipban/models.py @@ -19,6 +19,7 @@ def __init__( def run(self) -> List[LogsTableRow]: query_result: List[LogsTableRow] = self.get_blob_logs_query_result() + logging.info(f"Kusto query result: {query_result}") self.update_banned_ips(query_result) return query_result From 9b8282df0b2e377d84ce415ac595bf963addc0be Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 24 Jun 2024 17:08:29 -0700 Subject: [PATCH 38/41] UPDATE function settings --- pc-funcs.dev.env | 4 ++++ pcfuncs/ipban/config.py | 17 ++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/pc-funcs.dev.env b/pc-funcs.dev.env index 87bf075c..63b5f6ee 100644 --- a/pc-funcs.dev.env +++ b/pc-funcs.dev.env @@ -10,3 +10,7 @@ IMAGE_OUTPUT_STORAGE_URL="http://azurite:10000/devstoreaccount1/output/images" IMAGE_OUTPUT_ACCOUNT_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" IMAGE_API_ROOT_URL="https://planetarycomputer-staging.microsoft.com/api/data/v1" IMAGE_TILE_REQUEST_CONCURRENCY=2 + +STORAGE_ACCOUNT_URL=https://pctapisstagingsa.table.core.windows.net/ +BANNED_IP_TABLE=blobstoragebannedip +LOG_ANALYTICS_WORKSPACE_ID=78d48390-b6bb-49a9-b7fd-a86f6522e9c4 \ No newline at end of file diff --git a/pcfuncs/ipban/config.py b/pcfuncs/ipban/config.py index 1fe518ee..cd7b9a5d 100644 --- a/pcfuncs/ipban/config.py +++ b/pcfuncs/ipban/config.py @@ -1,18 +1,17 @@ # config.py -from pydantic import BaseSettings +from pydantic import BaseSettings, Field class Settings(BaseSettings): - # Constants related to Azure Table Storage - storage_account_url: str = "https://pctapisstagingsa.table.core.windows.net/" - banned_ip_table: str = "blobstoragebannedip" - - # Log Analytics Workspace: pc-api-loganalytics - log_analytics_workspace_id: str = "78d48390-b6bb-49a9-b7fd-a86f6522e9c4" + storage_account_url: str = Field(env="STORAGE_ACCOUNT_URL") + banned_ip_table: str = Field(env="BANNED_IP_TABLE") + log_analytics_workspace_id: str = Field(env="LOG_ANALYTICS_WORKSPACE_ID") # Time and threshold settings - time_window_in_hours: int = 24 - threshold_read_count_in_gb: int = 5120 + time_window_in_hours: int = Field(default=24, env="TIME_WINDOW_IN_HOURS") + threshold_read_count_in_gb: int = Field( + default=5120, env="THRESHOLD_READ_COUNT_IN_GB" + ) # Create a global settings instance From 58a6c196d7d2593c1f66b34efa3fd7e6cfb108ed Mon Sep 17 00:00:00 2001 From: elay Date: Mon, 24 Jun 2024 17:29:44 -0700 Subject: [PATCH 39/41] suppress mypy warning --- pcfuncs/ipban/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pcfuncs/ipban/config.py b/pcfuncs/ipban/config.py index cd7b9a5d..4261461d 100644 --- a/pcfuncs/ipban/config.py +++ b/pcfuncs/ipban/config.py @@ -15,4 +15,4 @@ class Settings(BaseSettings): # Create a global settings instance -settings = Settings() +settings = Settings() # type: ignore From 7c832c2a3441f2105b0942a70d57f49f574f9942 Mon Sep 17 00:00:00 2001 From: elay Date: Tue, 25 Jun 2024 14:27:02 -0700 Subject: [PATCH 40/41] update env variables for function app --- deployment/terraform/resources/functions.tf | 5 +++++ deployment/terraform/resources/variables.tf | 12 ++++++++++++ deployment/terraform/staging/main.tf | 3 +++ 3 files changed, 20 insertions(+) diff --git a/deployment/terraform/resources/functions.tf b/deployment/terraform/resources/functions.tf index fb1579db..add58ee1 100644 --- a/deployment/terraform/resources/functions.tf +++ b/deployment/terraform/resources/functions.tf @@ -41,6 +41,11 @@ resource "azurerm_function_app" "pcfuncs" { "IMAGE_OUTPUT_STORAGE_URL" = var.image_output_storage_url, "IMAGE_API_ROOT_URL" = var.funcs_data_api_url, "IMAGE_TILE_REQUEST_CONCURRENCY" = tostring(var.funcs_tile_request_concurrency), + + # IPban function + "STORAGE_ACCOUNT_URL" = var.func_storage_account_url, + "BANNED_IP_TABLE" = var.banned_ip_table, + "LOG_ANALYTICS_WORKSPACE_ID" = var.prod_log_analytics_workspace_id, } os_type = "linux" diff --git a/deployment/terraform/resources/variables.tf b/deployment/terraform/resources/variables.tf index 1604d6c2..e83b5f57 100644 --- a/deployment/terraform/resources/variables.tf +++ b/deployment/terraform/resources/variables.tf @@ -132,6 +132,18 @@ variable "prod_log_analytics_workspace_name" { type = string } +variable "prod_log_analytics_workspace_id" { + type = string +} + +variable "banned_ip_table" { + type = string +} + +variable "func_storage_account_url" { + type = string +} + # ----------------- # Local variables diff --git a/deployment/terraform/staging/main.tf b/deployment/terraform/staging/main.tf index 2dd7ec7e..b9f3ce92 100644 --- a/deployment/terraform/staging/main.tf +++ b/deployment/terraform/staging/main.tf @@ -23,6 +23,9 @@ module "resources" { image_output_storage_url = "https://pcfilestest.blob.core.windows.net/output/images" prod_log_analytics_workspace_name = "pc-api-loganalytics" + prod_log_analytics_workspace_id = "78d48390-b6bb-49a9-b7fd-a86f6522e9c4" + func_storage_account_url = "https://pctapisstagingsa.table.core.windows.net/" + banned_ip_table = "blobstoragebannedip" } terraform { From e0313eb430617a1b8d4e90bb5e64148b3072eb50 Mon Sep 17 00:00:00 2001 From: elay Date: Tue, 25 Jun 2024 14:53:28 -0700 Subject: [PATCH 41/41] typo --- deployment/terraform/resources/functions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/terraform/resources/functions.tf b/deployment/terraform/resources/functions.tf index add58ee1..764a3e45 100644 --- a/deployment/terraform/resources/functions.tf +++ b/deployment/terraform/resources/functions.tf @@ -42,7 +42,7 @@ resource "azurerm_function_app" "pcfuncs" { "IMAGE_API_ROOT_URL" = var.funcs_data_api_url, "IMAGE_TILE_REQUEST_CONCURRENCY" = tostring(var.funcs_tile_request_concurrency), - # IPban function + # IPBan function "STORAGE_ACCOUNT_URL" = var.func_storage_account_url, "BANNED_IP_TABLE" = var.banned_ip_table, "LOG_ANALYTICS_WORKSPACE_ID" = var.prod_log_analytics_workspace_id,