From 84a6ec93c6107c1ab4c47c41f7cf9e25989d925c Mon Sep 17 00:00:00 2001 From: Travis Bowen Date: Thu, 3 Apr 2025 11:56:08 -0700 Subject: [PATCH] PySpark Update AWS Region --- regtests/t_pyspark/src/iceberg_spark.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/regtests/t_pyspark/src/iceberg_spark.py b/regtests/t_pyspark/src/iceberg_spark.py index cd0c4ade52..339888ef8d 100644 --- a/regtests/t_pyspark/src/iceberg_spark.py +++ b/regtests/t_pyspark/src/iceberg_spark.py @@ -18,6 +18,7 @@ # """Spark connector with different catalog types.""" +import os from typing import Any, Dict, List, Optional, Union from pyspark.errors import PySparkRuntimeError @@ -43,7 +44,7 @@ def __init__( self, bearer_token: str = None, credentials: str = None, - aws_region: str = "us-west-2", + aws_region: str = None, catalog_name: str = None, polaris_url: str = None, realm: str = 'POLARIS' @@ -51,7 +52,8 @@ def __init__( """Constructor for Iceberg Spark session. Sets the member variables.""" self.bearer_token = bearer_token self.credentials = credentials - self.aws_region = aws_region + if aws_region is None: + self.aws_region = os.environ.get('AWS_REGION', 'us-west-2') self.catalog_name = catalog_name self.polaris_url = polaris_url self.realm = realm