From 30ff635a800996eb456a1ee25953d68445ffcfb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Korn=C3=A9l=20Regius?= Date: Sun, 20 Dec 2020 19:24:06 +0000 Subject: [PATCH 1/5] Cloud Translation, Natural language, Vision, Video intelligence GCP integrations. --- Dockerfile | 4 + patches/kaggle_gcp.py | 141 ++++++++++++++++++++++++++++--- patches/kaggle_secrets.py | 16 ++++ patches/sitecustomize.py | 25 +++++- tests/test_natural_language.py | 67 +++++++++++++++ tests/test_translation.py | 131 ++++++++++++++++++++++++++++ tests/test_user_secrets.py | 36 ++++++++ tests/test_video_intelligence.py | 66 +++++++++++++++ tests/test_vision.py | 66 +++++++++++++++ 9 files changed, 538 insertions(+), 14 deletions(-) create mode 100644 tests/test_natural_language.py create mode 100644 tests/test_translation.py create mode 100644 tests/test_video_intelligence.py create mode 100644 tests/test_vision.py diff --git a/Dockerfile b/Dockerfile index 6788bcae..e3877f40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -268,6 +268,10 @@ RUN pip install --upgrade cython && \ # which is loaded at startup. pip install google-cloud-bigquery==1.12.1 && \ pip install google-cloud-storage && \ + pip install google-cloud-translate==3.* && \ + pip install google-cloud-language==2.* && \ + pip install google-cloud-videointelligence==2.* && \ + pip install google-cloud-vision==2.* && \ pip install ortools && \ pip install scattertext && \ # Pandas data reader diff --git a/patches/kaggle_gcp.py b/patches/kaggle_gcp.py index 39c0168e..e2302887 100644 --- a/patches/kaggle_gcp.py +++ b/patches/kaggle_gcp.py @@ -1,6 +1,6 @@ import os import inspect -from google.auth import credentials +from google.auth import credentials, environment_vars from google.auth.exceptions import RefreshError from google.api_core.gapic_v1.client_info import ClientInfo from google.cloud import bigquery @@ -22,7 +22,7 @@ def get_integrations(): target = GcpTarget[integration.upper()] kernel_integrations.add_integration(target) except KeyError as e: - Log.error(f"Unknown integration target: {e}") + Log.error(f"Unknown integration target: {integration.upper()}") return kernel_integrations @@ -45,6 +45,17 @@ def has_gcs(self): def has_automl(self): return GcpTarget.AUTOML in self.integrations + def has_translation(self): + return GcpTarget.TRANSLATION in self.integrations + + def has_natural_language(self): + return GcpTarget.NATURAL_LANGUAGE in self.integrations + + def has_video_intelligence(self): + return GcpTarget.VIDEO_INTELLIGENCE in self.integrations + + def has_vision(self): + return GcpTarget.VISION in self.integrations class KaggleKernelCredentials(credentials.Credentials): """Custom Credentials used to authenticate using the Kernel's connected OAuth account. @@ -65,6 +76,14 @@ def refresh(self, request): self.token, self.expiry = client._get_gcs_access_token() elif self.target == GcpTarget.AUTOML: self.token, self.expiry = client._get_automl_access_token() + elif self.target == GcpTarget.TRANSLATION: + self.token, self.expiry = client._get_translation_access_token() + elif self.target == GcpTarget.NATURAL_LANGUAGE: + self.token, self.expiry = client._get_natural_language_access_token() + elif self.target == GcpTarget.VIDEO_INTELLIGENCE: + self.token, self.expiry = client._get_video_intelligence_access_token() + elif self.target == GcpTarget.VISION: + self.token, self.expiry = client._get_vision_access_token() except ConnectionError as e: Log.error(f"Connection error trying to refresh access token: {e}") print("There was a connection error trying to fetch the access token. " @@ -78,6 +97,12 @@ def refresh(self, request): f"Please ensure you have selected a {self.target.service} account in the Notebook Add-ons menu.") raise RefreshError('Unable to refresh access token.') from e +class KaggleKernelWithProjetCredentials(KaggleKernelCredentials): + """ Wrapper Kaggle Credentials with quota_project_id. + """ + def __init__(self, parentCredential=None, quota_project_id=None): + super().__init__(target=parentCredential.target) + self._quota_project_id=quota_project_id class _DataProxyConnection(Connection): """Custom Connection class used to proxy the BigQuery client to Kaggle's data proxy.""" @@ -122,13 +147,16 @@ def __init__(self, *args, **kwargs): def has_been_monkeypatched(method): return "kaggle_gcp" in inspect.getsourcefile(method) +def is_user_secrets_token_set(): + return "KAGGLE_USER_SECRETS_TOKEN" in os.environ + +def is_proxy_token_set(): + return "KAGGLE_DATA_PROXY_TOKEN" in os.environ + def init_bigquery(): - from google.auth import environment_vars from google.cloud import bigquery - is_proxy_token_set = "KAGGLE_DATA_PROXY_TOKEN" in os.environ - is_user_secrets_token_set = "KAGGLE_USER_SECRETS_TOKEN" in os.environ - if not (is_proxy_token_set or is_user_secrets_token_set): + if not (is_proxy_token_set() or is_user_secrets_token_set()): return bigquery # If this Notebook has bigquery integration on startup, preload the Kaggle Credentials @@ -185,14 +213,24 @@ def patched_init(self, *args, **kwargs): specified_credentials = kwargs.get('credentials') if specified_credentials is None: Log.info("No credentials specified, using KaggleKernelCredentials.") - kwargs['credentials'] = kaggle_kernel_credentials + # Some GCP services demand the billing and target project must be the same. + # To avoid using default service account based credential as caller credential + # user need to provide ClientOptions with quota_project_id: + # srv.Client(client_options=client_options.ClientOptions(quota_project_id="YOUR PROJECT")) + client_options=kwargs.get('client_options') + if client_options != None and client_options.quota_project_id != None: + kwargs['credentials'] = KaggleKernelWithProjetCredentials( + parentCredential = kaggle_kernel_credentials, + quota_project_id = client_options.quota_project_id) + else: + kwargs['credentials'] = kaggle_kernel_credentials kwargs['client_info'] = set_kaggle_user_agent(kwargs.get('client_info')) - return client_init(self, *args, **kwargs) if (not has_been_monkeypatched(client_klass.__init__)): client_klass.__init__ = patched_init + Log.info(f"Client patched: {client_klass}") def set_kaggle_user_agent(client_info: ClientInfo): # Add kaggle client user agent in order to attribute usage. @@ -203,9 +241,8 @@ def set_kaggle_user_agent(client_info: ClientInfo): return client_info def init_gcs(): - is_user_secrets_token_set = "KAGGLE_USER_SECRETS_TOKEN" in os.environ from google.cloud import storage - if not is_user_secrets_token_set: + if not is_user_secrets_token_set(): return storage from kaggle_gcp import get_integrations @@ -220,9 +257,8 @@ def init_gcs(): return storage def init_automl(): - is_user_secrets_token_set = "KAGGLE_USER_SECRETS_TOKEN" in os.environ from google.cloud import automl, automl_v1beta1 - if not is_user_secrets_token_set: + if not is_user_secrets_token_set(): return from kaggle_gcp import get_integrations @@ -251,10 +287,91 @@ def init_automl(): # the TablesClient is GA. monkeypatch_client(automl_v1beta1.TablesClient, kaggle_kernel_credentials) +def init_translation_v2(): + from google.cloud import translate_v2 + if not is_user_secrets_token_set(): + return translate_v2 + + from kaggle_gcp import get_integrations + if not get_integrations().has_translation(): + return translate_v2 + from kaggle_secrets import GcpTarget + kernel_credentials = KaggleKernelCredentials(target=GcpTarget.TRANSLATION) + monkeypatch_client(translate_v2.Client,kernel_credentials) + return translate_v2 + +def init_translation_v3(): + # Translate v3 exposes different client than translate v2. + from google.cloud import translate_v3 + if not is_user_secrets_token_set(): + return translate_v3 + + from kaggle_gcp import get_integrations + if not get_integrations().has_translation(): + return translate_v3 + from kaggle_secrets import GcpTarget + kernel_credentials = KaggleKernelCredentials(target=GcpTarget.TRANSLATION) + monkeypatch_client(translate_v3.TranslationServiceClient,kernel_credentials) + return translate_v3 + +def init_natural_language(): + from google.cloud import language + if not is_user_secrets_token_set(): + return language + + from kaggle_gcp import get_integrations + if not get_integrations().has_natural_language(): + return language + + from kaggle_secrets import GcpTarget + kernel_credentials = KaggleKernelCredentials(target=GcpTarget.NATURAL_LANGUAGE) + monkeypatch_client(language.LanguageServiceClient,kernel_credentials) + monkeypatch_client(language.LanguageServiceAsyncClient,kernel_credentials) + return language + +def init_video_intelligence(): + from google.cloud import videointelligence + if not is_user_secrets_token_set(): + return videointelligence + + from kaggle_gcp import get_integrations + if not get_integrations().has_video_intelligence(): + return videointelligence + + from kaggle_secrets import GcpTarget + kernel_credentials = KaggleKernelCredentials(target=GcpTarget.VIDEO_INTELLIGENCE) + monkeypatch_client( + videointelligence.VideoIntelligenceServiceClient, + kernel_credentials) + monkeypatch_client( + videointelligence.VideoIntelligenceServiceAsyncClient, + kernel_credentials) + return videointelligence + +def init_vision(): + from google.cloud import vision + if not is_user_secrets_token_set(): + return vision + + from kaggle_gcp import get_integrations + if not get_integrations().has_vision(): + return vision + + from kaggle_secrets import GcpTarget + kernel_credentials = KaggleKernelCredentials(target=GcpTarget.VISION) + monkeypatch_client(vision.ImageAnnotatorClient,kernel_credentials) + monkeypatch_client(vision.ImageAnnotatorAsyncClient,kernel_credentials) + return vision + def init(): init_bigquery() init_gcs() init_automl() + init_translation_v2() + init_translation_v3() + init_natural_language() + init_video_intelligence() + init_vision() # We need to initialize the monkeypatching of the client libraries # here since there is a circular dependency between our import hook version diff --git a/patches/kaggle_secrets.py b/patches/kaggle_secrets.py index 8f0d3eb3..0aa7c01e 100644 --- a/patches/kaggle_secrets.py +++ b/patches/kaggle_secrets.py @@ -24,6 +24,10 @@ class GcpTarget(Enum): BIGQUERY = (1, "BigQuery") GCS = (2, "Google Cloud Storage") AUTOML = (3, "Cloud AutoML") + TRANSLATION = (4, "Cloud Translation") + NATURAL_LANGUAGE = (5, "Cloud Natural Language") + VIDEO_INTELLIGENCE = (6, "Cloud Video Intelligence") + VISION = (7, "Cloud Vision") def __init__(self, target, service): self._target = target @@ -154,6 +158,18 @@ def _get_gcs_access_token(self) -> Tuple[str, Optional[datetime]]: def _get_automl_access_token(self) -> Tuple[str, Optional[datetime]]: return self._get_access_token(GcpTarget.AUTOML) + def _get_translation_access_token(self) -> Tuple[str, Optional[datetime]]: + return self._get_access_token(GcpTarget.TRANSLATION) + + def _get_natural_language_access_token(self) -> Tuple[str, Optional[datetime]]: + return self._get_access_token(GcpTarget.NATURAL_LANGUAGE) + + def _get_video_intelligence_access_token(self) -> Tuple[str, Optional[datetime]]: + return self._get_access_token(GcpTarget.VIDEO_INTELLIGENCE) + + def _get_vision_access_token(self) -> Tuple[str, Optional[datetime]]: + return self._get_access_token(GcpTarget.VISION) + def _get_access_token(self, target: GcpTarget) -> Tuple[str, Optional[datetime]]: request_body = { 'Target': target.target diff --git a/patches/sitecustomize.py b/patches/sitecustomize.py index 04774a11..fa5b274d 100644 --- a/patches/sitecustomize.py +++ b/patches/sitecustomize.py @@ -7,7 +7,20 @@ import importlib.machinery class GcpModuleFinder(importlib.abc.MetaPathFinder): - _MODULES = ['google.cloud.bigquery', 'google.cloud.storage', 'google.cloud.automl_v1beta1'] + _MODULES = [ + 'google.cloud.bigquery', + 'google.cloud.storage', + 'google.cloud.automl_v1beta1', + 'google.cloud.translate', + 'google.cloud.translate_v2', + 'google.cloud.translate_v3', + 'google.cloud.language', + 'google.cloud.language_v1', + 'google.cloud.videointelligence', + 'google.cloud.videointelligence_v1', + 'google.cloud.vision', + 'google.cloud.vision_v1', + ] _KAGGLE_GCP_PATH = 'kaggle_gcp.py' def __init__(self): pass @@ -41,6 +54,15 @@ def create_module(self, spec): 'google.cloud.bigquery': kaggle_gcp.init_bigquery, 'google.cloud.storage': kaggle_gcp.init_gcs, 'google.cloud.automl_v1beta1': kaggle_gcp.init_automl, + 'google.cloud.translate': kaggle_gcp.init_translation_v3, + 'google.cloud.translate_v2': kaggle_gcp.init_translation_v2, + 'google.cloud.translate_v3': kaggle_gcp.init_translation_v3, + 'google.cloud.language': kaggle_gcp.init_natural_language, + 'google.cloud.language_v1': kaggle_gcp.init_natural_language, + 'google.cloud.videointelligence': kaggle_gcp.init_video_intelligence, + 'google.cloud.videointelligence_v1': kaggle_gcp.init_video_intelligence, + 'google.cloud.vision': kaggle_gcp.init_vision, + 'google.cloud.vision_v1': kaggle_gcp.init_vision } monkeypatch_gcp_module = _LOADERS[spec.name]() return monkeypatch_gcp_module @@ -48,6 +70,5 @@ def create_module(self, spec): def exec_module(self, module): pass - if not hasattr(sys, 'frozen'): sys.meta_path.insert(0, GcpModuleFinder()) diff --git a/tests/test_natural_language.py b/tests/test_natural_language.py new file mode 100644 index 00000000..0ae6f6ed --- /dev/null +++ b/tests/test_natural_language.py @@ -0,0 +1,67 @@ +import unittest +import inspect + +from unittest.mock import Mock, patch + +from kaggle_gcp import KaggleKernelCredentials, init_natural_language +from test.support import EnvironmentVarGuard +from google.cloud import language + +def _make_credentials(): + import google.auth.credentials + return Mock(spec=google.auth.credentials.Credentials) + +class TestCloudNaturalLanguage(unittest.TestCase): + class FakeClient: + def __init__(self, credentials=None, client_info=None, **kwargs): + self.credentials = credentials + + class FakeConnection(): + def __init__(self, user_agent): + self.user_agent = user_agent + if (client_info is not None): + self._connection = FakeConnection(client_info.user_agent) + + @patch("google.cloud.language.LanguageServiceClient", new=FakeClient) + def test_default_credentials(self): + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE') + with env: + init_natural_language() + client = language.LanguageServiceClient() + self.assertIsNotNone(client.credentials) + self.assertIsInstance(client.credentials, KaggleKernelCredentials) + + @patch("google.cloud.language.LanguageServiceClient", new=FakeClient) + def test_user_provided_credentials(self): + credentials = _make_credentials() + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE') + with env: + init_natural_language() + client = language.LanguageServiceClient(credentials=credentials) + self.assertIsNotNone(client.credentials) + self.assertNotIsInstance(client.credentials, KaggleKernelCredentials) + + + def test_monkeypatching_succeed(self): + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + with env: + init_natural_language() + client = language.LanguageServiceClient.__init__ + self.assertTrue("kaggle_gcp" in inspect.getsourcefile(client)) + + def test_monkeypatching_idempotent(self): + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE') + with env: + init_natural_language() + client1 = language.LanguageServiceClient.__init__ + init_natural_language() + client2 = language.LanguageServiceClient.__init__ + self.assertEqual(client1, client2) diff --git a/tests/test_translation.py b/tests/test_translation.py new file mode 100644 index 00000000..83618a14 --- /dev/null +++ b/tests/test_translation.py @@ -0,0 +1,131 @@ +import unittest +import inspect + +from unittest.mock import Mock, patch + +from kaggle_gcp import KaggleKernelCredentials, KaggleKernelWithProjetCredentials, init_translation_v2, init_translation_v3 +from test.support import EnvironmentVarGuard +from google.api_core import client_options +from google.cloud import translate, translate_v2 + +def _make_credentials(): + import google.auth.credentials + return Mock(spec=google.auth.credentials.Credentials) + +class TestCloudTranslation(unittest.TestCase): + class FakeClient: + def __init__(self, credentials=None, client_info=None, client_options=None, **kwargs): + self.credentials = credentials + self.client_options = client_options + + class FakeConnection(): + def __init__(self, user_agent): + self.user_agent = user_agent + if (client_info is not None): + self._connection = FakeConnection(client_info.user_agent) + + @patch("google.cloud.translate_v2.Client", new=FakeClient) + def test_default_credentials_v2(self): + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + with env: + init_translation_v2() + client = translate_v2.Client() + self.assertIsNotNone(client.credentials) + self.assertIsInstance(client.credentials, KaggleKernelCredentials) + + + @patch("google.cloud.translate_v2.Client", new=FakeClient) + def test_user_provided_credentials_v2(self): + credentials = _make_credentials() + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + with env: + init_translation_v2() + client = translate_v2.Client(credentials=credentials) + self.assertIsNotNone(client.credentials) + self.assertNotIsInstance(client.credentials, KaggleKernelCredentials) + + @patch("google.cloud.translate.TranslationServiceClient", new=FakeClient) + def test_default_credentials_v3(self): + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + with env: + init_translation_v3() + client = translate.TranslationServiceClient() + self.assertIsNotNone(client.credentials) + self.assertIsInstance(client.credentials, KaggleKernelCredentials) + + + @patch("google.cloud.translate.TranslationServiceClient", new=FakeClient) + def test_user_provided_credentials_v3(self): + credentials = _make_credentials() + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + with env: + init_translation_v3() + client = translate.TranslationServiceClient(credentials=credentials) + self.assertIsNotNone(client.credentials) + self.assertNotIsInstance(client.credentials, KaggleKernelCredentials) + + + def test_monkeypatching_succeed(self): + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + with env: + init_translation_v2() + init_translation_v3() + # Translation V2 + client2 = translate_v2.Client.__init__ + self.assertTrue("kaggle_gcp" in inspect.getsourcefile(client2)) + # Translation V3 + client3 = translate.TranslationServiceClient.__init__ + self.assertTrue("kaggle_gcp" in inspect.getsourcefile(client3)) + + def test_monkeypatching_idempotent(self): + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + with env: + init_translation_v2() + init_translation_v3() + # Translation V2 + client2_1 = translate_v2.client.Client.__init__ + # Translation V3 + client3_1 = translate.TranslationServiceClient.__init__ + + init_translation_v2() + init_translation_v3() + + client2_2 = translate_v2.Client.__init__ + client3_2 = translate.TranslationServiceClient.__init__ + self.assertEqual(client2_1, client2_2) + self.assertEqual(client3_1, client3_2) + + @patch("google.cloud.translate.TranslationServiceClient", new=FakeClient) + def test_client_credential_uniqueness_v3(self): + """ + Client instance must use unique KaggleKernelWithProjetCredentials with quota_project_id + when client_options.quota_project_id provided. (even if quota_project_id is same) + """ + credentials = _make_credentials() + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + with env: + init_translation_v3() + client1 = translate.TranslationServiceClient() + client2 = translate.TranslationServiceClient(client_options=client_options.ClientOptions(quota_project_id="_doesn't_matter")) + client3 = translate.TranslationServiceClient(client_options=client_options.ClientOptions(quota_project_id="_doesn't_matter2")) + self.assertIsNotNone(client1.credentials) + self.assertIsNotNone(client2.credentials) + self.assertIsNotNone(client3.credentials) + self.assertIsInstance(client2.credentials, KaggleKernelWithProjetCredentials) + self.assertIsInstance(client3.credentials, KaggleKernelWithProjetCredentials) + self.assertNotEqual(client1.credentials, client2.credentials) + self.assertNotEqual(client2.credentials, client3.credentials) diff --git a/tests/test_user_secrets.py b/tests/test_user_secrets.py index fdaa58b2..7f71ccf4 100644 --- a/tests/test_user_secrets.py +++ b/tests/test_user_secrets.py @@ -178,12 +178,48 @@ def call_get_gcs_access_token(): client = UserSecretsClient() secret_response = client._get_gcs_access_token() self.assertEqual(secret_response, (secret, now + timedelta(seconds=3600))) + def call_get_automl_access_token(): + client = UserSecretsClient() + secret_response = client._get_automl_access_token() + self.assertEqual(secret_response, (secret, now + timedelta(seconds=3600))) + def call_get_translation_access_token(): + client = UserSecretsClient() + secret_response = client._get_translation_access_token() + self.assertEqual(secret_response, (secret, now + timedelta(seconds=3600))) + def call_get_natural_lang_access_token(): + client = UserSecretsClient() + secret_response = client._get_natural_language_access_token() + self.assertEqual(secret_response, (secret, now + timedelta(seconds=3600))) + def call_get_video_intell_access_token(): + client = UserSecretsClient() + secret_response = client._get_video_intelligence_access_token() + self.assertEqual(secret_response, (secret, now + timedelta(seconds=3600))) + def call_get_vision_access_token(): + client = UserSecretsClient() + secret_response = client._get_vision_access_token() + self.assertEqual(secret_response, (secret, now + timedelta(seconds=3600))) + self._test_client(call_get_bigquery_access_token, '/requests/GetUserSecretRequest', {'Target': GcpTarget.BIGQUERY.target}, secret=secret) self._test_client(call_get_gcs_access_token, '/requests/GetUserSecretRequest', {'Target': GcpTarget.GCS.target}, secret=secret) + self._test_client(call_get_automl_access_token, + '/requests/GetUserSecretRequest', {'Target': GcpTarget.AUTOML.target}, + secret=secret) + self._test_client(call_get_translation_access_token, + '/requests/GetUserSecretRequest', {'Target': GcpTarget.TRANSLATION.target}, + secret=secret) + self._test_client(call_get_natural_lang_access_token, + '/requests/GetUserSecretRequest', {'Target': GcpTarget.NATURAL_LANGUAGE.target}, + secret=secret) + self._test_client(call_get_video_intell_access_token, + '/requests/GetUserSecretRequest', {'Target': GcpTarget.VIDEO_INTELLIGENCE.target}, + secret=secret) + self._test_client(call_get_vision_access_token, + '/requests/GetUserSecretRequest', {'Target': GcpTarget.VISION.target}, + secret=secret) def test_get_access_token_handles_unsuccessful(self): def call_get_access_token(): diff --git a/tests/test_video_intelligence.py b/tests/test_video_intelligence.py new file mode 100644 index 00000000..856aefdf --- /dev/null +++ b/tests/test_video_intelligence.py @@ -0,0 +1,66 @@ +import unittest +import inspect + +from unittest.mock import Mock, patch + +from kaggle_gcp import KaggleKernelCredentials, init_video_intelligence +from test.support import EnvironmentVarGuard +from google.cloud import videointelligence + +def _make_credentials(): + import google.auth.credentials + return Mock(spec=google.auth.credentials.Credentials) + +class TestCloudVideoIntelligence(unittest.TestCase): + class FakeClient: + def __init__(self, credentials=None, client_info=None, **kwargs): + self.credentials = credentials + + class FakeConnection(): + def __init__(self, user_agent): + self.user_agent = user_agent + if (client_info is not None): + self._connection = FakeConnection(client_info.user_agent) + + @patch("google.cloud.videointelligence.VideoIntelligenceServiceClient", new=FakeClient) + def test_default_credentials(self): + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VIDEO_INTELLIGENCE') + with env: + init_video_intelligence() + client = videointelligence.VideoIntelligenceServiceClient() + self.assertIsNotNone(client.credentials) + self.assertIsInstance(client.credentials, KaggleKernelCredentials) + + @patch("google.cloud.videointelligence.VideoIntelligenceServiceClient", new=FakeClient) + def test_user_provided_credentials(self): + credentials = _make_credentials() + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VIDEO_INTELLIGENCE') + with env: + init_video_intelligence() + client = videointelligence.VideoIntelligenceServiceClient(credentials=credentials) + self.assertNotIsInstance(client.credentials, KaggleKernelCredentials) + self.assertIsNotNone(client.credentials) + + def test_monkeypatching_succeed(self): + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + with env: + init_video_intelligence() + client = videointelligence.VideoIntelligenceServiceClient.__init__ + self.assertTrue("kaggle_gcp" in inspect.getsourcefile(client)) + + def test_monkeypatching_idempotent(self): + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VIDEO_INTELLIGENCE') + with env: + init_video_intelligence() + client1 = videointelligence.VideoIntelligenceServiceClient.__init__ + init_video_intelligence() + client2 = videointelligence.VideoIntelligenceServiceClient.__init__ + self.assertEqual(client1, client2) diff --git a/tests/test_vision.py b/tests/test_vision.py new file mode 100644 index 00000000..0cd0c295 --- /dev/null +++ b/tests/test_vision.py @@ -0,0 +1,66 @@ +import unittest +import inspect + +from unittest.mock import Mock, patch + +from kaggle_gcp import KaggleKernelCredentials, init_vision +from test.support import EnvironmentVarGuard +from google.cloud import vision + +def _make_credentials(): + import google.auth.credentials + return Mock(spec=google.auth.credentials.Credentials) + +class TestCloudVision(unittest.TestCase): + class FakeClient: + def __init__(self, credentials=None, client_info=None, **kwargs): + self.credentials = credentials + + class FakeConnection(): + def __init__(self, user_agent): + self.user_agent = user_agent + if (client_info is not None): + self._connection = FakeConnection(client_info.user_agent) + + @patch("google.cloud.vision.ImageAnnotatorClient", new=FakeClient) + def test_default_credentials(self): + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VISION') + with env: + init_vision() + client = vision.ImageAnnotatorClient() + self.assertIsNotNone(client.credentials) + self.assertIsInstance(client.credentials, KaggleKernelCredentials) + + @patch("google.cloud.vision.ImageAnnotatorClient", new=FakeClient) + def test_user_provided_credentials(self): + credentials = _make_credentials() + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VISION') + with env: + init_vision() + client = vision.ImageAnnotatorClient(credentials=credentials) + self.assertNotIsInstance(client.credentials, KaggleKernelCredentials) + self.assertIsNotNone(client.credentials) + + def test_monkeypatching_succeed(self): + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + with env: + init_vision() + client = vision.ImageAnnotatorClient.__init__ + self.assertTrue("kaggle_gcp" in inspect.getsourcefile(client)) + + def test_monkeypatching_idempotent(self): + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VISION') + with env: + init_vision() + client1 = vision.ImageAnnotatorClient.__init__ + init_vision() + client2 = vision.ImageAnnotatorClient.__init__ + self.assertEqual(client1, client2) From ef87f073a60c00100d969fe511e49df801157684 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Korn=C3=A9l=20Regius?= Date: Tue, 22 Dec 2020 14:13:52 +0000 Subject: [PATCH 2/5] Fixes for style guide --- patches/kaggle_gcp.py | 12 ++++++------ patches/sitecustomize.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/patches/kaggle_gcp.py b/patches/kaggle_gcp.py index e2302887..d084a0b7 100644 --- a/patches/kaggle_gcp.py +++ b/patches/kaggle_gcp.py @@ -297,7 +297,7 @@ def init_translation_v2(): return translate_v2 from kaggle_secrets import GcpTarget kernel_credentials = KaggleKernelCredentials(target=GcpTarget.TRANSLATION) - monkeypatch_client(translate_v2.Client,kernel_credentials) + monkeypatch_client(translate_v2.Client, kernel_credentials) return translate_v2 def init_translation_v3(): @@ -311,7 +311,7 @@ def init_translation_v3(): return translate_v3 from kaggle_secrets import GcpTarget kernel_credentials = KaggleKernelCredentials(target=GcpTarget.TRANSLATION) - monkeypatch_client(translate_v3.TranslationServiceClient,kernel_credentials) + monkeypatch_client(translate_v3.TranslationServiceClient, kernel_credentials) return translate_v3 def init_natural_language(): @@ -325,8 +325,8 @@ def init_natural_language(): from kaggle_secrets import GcpTarget kernel_credentials = KaggleKernelCredentials(target=GcpTarget.NATURAL_LANGUAGE) - monkeypatch_client(language.LanguageServiceClient,kernel_credentials) - monkeypatch_client(language.LanguageServiceAsyncClient,kernel_credentials) + monkeypatch_client(language.LanguageServiceClient, kernel_credentials) + monkeypatch_client(language.LanguageServiceAsyncClient, kernel_credentials) return language def init_video_intelligence(): @@ -359,8 +359,8 @@ def init_vision(): from kaggle_secrets import GcpTarget kernel_credentials = KaggleKernelCredentials(target=GcpTarget.VISION) - monkeypatch_client(vision.ImageAnnotatorClient,kernel_credentials) - monkeypatch_client(vision.ImageAnnotatorAsyncClient,kernel_credentials) + monkeypatch_client(vision.ImageAnnotatorClient, kernel_credentials) + monkeypatch_client(vision.ImageAnnotatorAsyncClient, kernel_credentials) return vision def init(): diff --git a/patches/sitecustomize.py b/patches/sitecustomize.py index fa5b274d..d01f3845 100644 --- a/patches/sitecustomize.py +++ b/patches/sitecustomize.py @@ -20,7 +20,7 @@ class GcpModuleFinder(importlib.abc.MetaPathFinder): 'google.cloud.videointelligence_v1', 'google.cloud.vision', 'google.cloud.vision_v1', - ] + ] _KAGGLE_GCP_PATH = 'kaggle_gcp.py' def __init__(self): pass From 6f9a5d68972e812cfaab501296d5b6ac45198d95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Korn=C3=A9l=20Regius?= Date: Mon, 11 Jan 2021 18:46:25 +0000 Subject: [PATCH 3/5] Simplified Cloud AI Platform integration --- patches/kaggle_gcp.py | 52 +++++++++------------------- patches/kaggle_secrets.py | 22 ++---------- tests/test_automl.py | 16 ++++----- tests/test_natural_language.py | 8 ++--- tests/test_tensorflow_credentials.py | 4 +-- tests/test_translation.py | 14 ++++---- tests/test_user_secrets.py | 20 +++-------- tests/test_video_intelligence.py | 8 ++--- tests/test_vision.py | 8 ++--- 9 files changed, 52 insertions(+), 100 deletions(-) diff --git a/patches/kaggle_gcp.py b/patches/kaggle_gcp.py index d084a0b7..cd21cb49 100644 --- a/patches/kaggle_gcp.py +++ b/patches/kaggle_gcp.py @@ -42,20 +42,8 @@ def has_bigquery(self): def has_gcs(self): return GcpTarget.GCS in self.integrations - def has_automl(self): - return GcpTarget.AUTOML in self.integrations - - def has_translation(self): - return GcpTarget.TRANSLATION in self.integrations - - def has_natural_language(self): - return GcpTarget.NATURAL_LANGUAGE in self.integrations - - def has_video_intelligence(self): - return GcpTarget.VIDEO_INTELLIGENCE in self.integrations - - def has_vision(self): - return GcpTarget.VISION in self.integrations + def has_cloudai(self): + return GcpTarget.CLOUDAI in self.integrations class KaggleKernelCredentials(credentials.Credentials): """Custom Credentials used to authenticate using the Kernel's connected OAuth account. @@ -74,16 +62,8 @@ def refresh(self, request): self.token, self.expiry = client.get_bigquery_access_token() elif self.target == GcpTarget.GCS: self.token, self.expiry = client._get_gcs_access_token() - elif self.target == GcpTarget.AUTOML: - self.token, self.expiry = client._get_automl_access_token() - elif self.target == GcpTarget.TRANSLATION: - self.token, self.expiry = client._get_translation_access_token() - elif self.target == GcpTarget.NATURAL_LANGUAGE: - self.token, self.expiry = client._get_natural_language_access_token() - elif self.target == GcpTarget.VIDEO_INTELLIGENCE: - self.token, self.expiry = client._get_video_intelligence_access_token() - elif self.target == GcpTarget.VISION: - self.token, self.expiry = client._get_vision_access_token() + elif self.target == GcpTarget.CLOUDAI: + self.token, self.expiry = client._get_cloudai_access_token() except ConnectionError as e: Log.error(f"Connection error trying to refresh access token: {e}") print("There was a connection error trying to fetch the access token. " @@ -262,12 +242,12 @@ def init_automl(): return from kaggle_gcp import get_integrations - if not get_integrations().has_automl(): + if not get_integrations().has_cloudai(): return from kaggle_secrets import GcpTarget from kaggle_gcp import KaggleKernelCredentials - kaggle_kernel_credentials = KaggleKernelCredentials(target=GcpTarget.AUTOML) + kaggle_kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI) # Patch the 2 GA clients: AutoMlClient and PreditionServiceClient monkeypatch_client(automl.AutoMlClient, kaggle_kernel_credentials) @@ -293,10 +273,10 @@ def init_translation_v2(): return translate_v2 from kaggle_gcp import get_integrations - if not get_integrations().has_translation(): + if not get_integrations().has_cloudai(): return translate_v2 from kaggle_secrets import GcpTarget - kernel_credentials = KaggleKernelCredentials(target=GcpTarget.TRANSLATION) + kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI) monkeypatch_client(translate_v2.Client, kernel_credentials) return translate_v2 @@ -307,10 +287,10 @@ def init_translation_v3(): return translate_v3 from kaggle_gcp import get_integrations - if not get_integrations().has_translation(): + if not get_integrations().has_cloudai(): return translate_v3 from kaggle_secrets import GcpTarget - kernel_credentials = KaggleKernelCredentials(target=GcpTarget.TRANSLATION) + kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI) monkeypatch_client(translate_v3.TranslationServiceClient, kernel_credentials) return translate_v3 @@ -320,11 +300,11 @@ def init_natural_language(): return language from kaggle_gcp import get_integrations - if not get_integrations().has_natural_language(): + if not get_integrations().has_cloudai(): return language from kaggle_secrets import GcpTarget - kernel_credentials = KaggleKernelCredentials(target=GcpTarget.NATURAL_LANGUAGE) + kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI) monkeypatch_client(language.LanguageServiceClient, kernel_credentials) monkeypatch_client(language.LanguageServiceAsyncClient, kernel_credentials) return language @@ -335,11 +315,11 @@ def init_video_intelligence(): return videointelligence from kaggle_gcp import get_integrations - if not get_integrations().has_video_intelligence(): + if not get_integrations().has_cloudai(): return videointelligence from kaggle_secrets import GcpTarget - kernel_credentials = KaggleKernelCredentials(target=GcpTarget.VIDEO_INTELLIGENCE) + kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI) monkeypatch_client( videointelligence.VideoIntelligenceServiceClient, kernel_credentials) @@ -354,11 +334,11 @@ def init_vision(): return vision from kaggle_gcp import get_integrations - if not get_integrations().has_vision(): + if not get_integrations().has_cloudai(): return vision from kaggle_secrets import GcpTarget - kernel_credentials = KaggleKernelCredentials(target=GcpTarget.VISION) + kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI) monkeypatch_client(vision.ImageAnnotatorClient, kernel_credentials) monkeypatch_client(vision.ImageAnnotatorAsyncClient, kernel_credentials) return vision diff --git a/patches/kaggle_secrets.py b/patches/kaggle_secrets.py index 0aa7c01e..c847a24a 100644 --- a/patches/kaggle_secrets.py +++ b/patches/kaggle_secrets.py @@ -23,11 +23,7 @@ class GcpTarget(Enum): """Enum class to store GCP targets.""" BIGQUERY = (1, "BigQuery") GCS = (2, "Google Cloud Storage") - AUTOML = (3, "Cloud AutoML") - TRANSLATION = (4, "Cloud Translation") - NATURAL_LANGUAGE = (5, "Cloud Natural Language") - VIDEO_INTELLIGENCE = (6, "Cloud Video Intelligence") - VISION = (7, "Cloud Vision") + CLOUDAI = (3, "Google Cloud AI Platform") def __init__(self, target, service): self._target = target @@ -155,20 +151,8 @@ def _write_gsutil_credentials_file(self, credentials) -> str: def _get_gcs_access_token(self) -> Tuple[str, Optional[datetime]]: return self._get_access_token(GcpTarget.GCS) - def _get_automl_access_token(self) -> Tuple[str, Optional[datetime]]: - return self._get_access_token(GcpTarget.AUTOML) - - def _get_translation_access_token(self) -> Tuple[str, Optional[datetime]]: - return self._get_access_token(GcpTarget.TRANSLATION) - - def _get_natural_language_access_token(self) -> Tuple[str, Optional[datetime]]: - return self._get_access_token(GcpTarget.NATURAL_LANGUAGE) - - def _get_video_intelligence_access_token(self) -> Tuple[str, Optional[datetime]]: - return self._get_access_token(GcpTarget.VIDEO_INTELLIGENCE) - - def _get_vision_access_token(self) -> Tuple[str, Optional[datetime]]: - return self._get_access_token(GcpTarget.VISION) + def _get_cloudai_access_token(self) -> Tuple[str, Optional[datetime]]: + return self._get_access_token(GcpTarget.CLOUDAI) def _get_access_token(self, target: GcpTarget) -> Tuple[str, Optional[datetime]]: request_body = { diff --git a/tests/test_automl.py b/tests/test_automl.py index 37571fe9..6a9e89e2 100644 --- a/tests/test_automl.py +++ b/tests/test_automl.py @@ -27,7 +27,7 @@ def test_user_provided_credentials(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_automl() client = automl.AutoMlClient(credentials=credentials) @@ -48,7 +48,7 @@ def test_tables_client_credentials(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_automl() tables_client = automl_v1beta1.TablesClient(credentials=credentials) @@ -58,7 +58,7 @@ def test_tables_client_credentials(self): def test_default_credentials_automl_client(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_automl() automl_client = automl.AutoMlClient() @@ -70,7 +70,7 @@ def test_default_credentials_automl_client(self): def test_default_credentials_automl_v1beta1_client(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_automl() automl_client = automl_v1beta1.AutoMlClient() @@ -82,7 +82,7 @@ def test_default_credentials_automl_v1beta1_client(self): def test_default_credentials_tables_client(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_automl() tables_client = automl_v1beta1.TablesClient() @@ -94,7 +94,7 @@ def test_default_credentials_tables_client(self): def test_default_credentials_prediction_client(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: prediction_client = automl.PredictionServiceClient() self.assertIsNotNone(prediction_client.credentials) @@ -105,7 +105,7 @@ def test_default_credentials_prediction_client(self): def test_default_credentials_prediction_v1beta1_client(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: prediction_client = automl_v1beta1.PredictionServiceClient() self.assertIsNotNone(prediction_client.credentials) @@ -115,7 +115,7 @@ def test_default_credentials_prediction_v1beta1_client(self): def test_monkeypatching_idempotent(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: client1 = automl.AutoMlClient.__init__ init_automl() diff --git a/tests/test_natural_language.py b/tests/test_natural_language.py index 0ae6f6ed..e2939013 100644 --- a/tests/test_natural_language.py +++ b/tests/test_natural_language.py @@ -26,7 +26,7 @@ def __init__(self, user_agent): def test_default_credentials(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_natural_language() client = language.LanguageServiceClient() @@ -38,7 +38,7 @@ def test_user_provided_credentials(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_natural_language() client = language.LanguageServiceClient(credentials=credentials) @@ -49,7 +49,7 @@ def test_user_provided_credentials(self): def test_monkeypatching_succeed(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_natural_language() client = language.LanguageServiceClient.__init__ @@ -58,7 +58,7 @@ def test_monkeypatching_succeed(self): def test_monkeypatching_idempotent(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_natural_language() client1 = language.LanguageServiceClient.__init__ diff --git a/tests/test_tensorflow_credentials.py b/tests/test_tensorflow_credentials.py index 42280446..628b35c1 100644 --- a/tests/test_tensorflow_credentials.py +++ b/tests/test_tensorflow_credentials.py @@ -13,7 +13,7 @@ def test_set_tensorflow_credential(self, mock_configure_gcs): credential = '{"client_id":"fake_client_id",' \ '"client_secret":"fake_client_secret",' \ '"refresh_token":"not a refresh token",' \ - '"type":"authorized_user"}'; + '"type":"authorized_user"}' env = EnvironmentVarGuard() env.set('HOME', '/tmp') @@ -22,7 +22,7 @@ def test_set_tensorflow_credential(self, mock_configure_gcs): # These need to be set to make UserSecretsClient happy, but aren't # pertinent to this test. env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') user_secrets = UserSecretsClient() user_secrets.set_tensorflow_credential(credential) diff --git a/tests/test_translation.py b/tests/test_translation.py index 83618a14..a55132c9 100644 --- a/tests/test_translation.py +++ b/tests/test_translation.py @@ -28,7 +28,7 @@ def __init__(self, user_agent): def test_default_credentials_v2(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_translation_v2() client = translate_v2.Client() @@ -41,7 +41,7 @@ def test_user_provided_credentials_v2(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_translation_v2() client = translate_v2.Client(credentials=credentials) @@ -52,7 +52,7 @@ def test_user_provided_credentials_v2(self): def test_default_credentials_v3(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_translation_v3() client = translate.TranslationServiceClient() @@ -65,7 +65,7 @@ def test_user_provided_credentials_v3(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_translation_v3() client = translate.TranslationServiceClient(credentials=credentials) @@ -76,7 +76,7 @@ def test_user_provided_credentials_v3(self): def test_monkeypatching_succeed(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_translation_v2() init_translation_v3() @@ -90,7 +90,7 @@ def test_monkeypatching_succeed(self): def test_monkeypatching_idempotent(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_translation_v2() init_translation_v3() @@ -116,7 +116,7 @@ def test_client_credential_uniqueness_v3(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_translation_v3() client1 = translate.TranslationServiceClient() diff --git a/tests/test_user_secrets.py b/tests/test_user_secrets.py index 7f71ccf4..606a5c8c 100644 --- a/tests/test_user_secrets.py +++ b/tests/test_user_secrets.py @@ -178,9 +178,9 @@ def call_get_gcs_access_token(): client = UserSecretsClient() secret_response = client._get_gcs_access_token() self.assertEqual(secret_response, (secret, now + timedelta(seconds=3600))) - def call_get_automl_access_token(): + def call_get_cloudai_access_token(): client = UserSecretsClient() - secret_response = client._get_automl_access_token() + secret_response = client._get_cloudai_access_token() self.assertEqual(secret_response, (secret, now + timedelta(seconds=3600))) def call_get_translation_access_token(): client = UserSecretsClient() @@ -205,20 +205,8 @@ def call_get_vision_access_token(): self._test_client(call_get_gcs_access_token, '/requests/GetUserSecretRequest', {'Target': GcpTarget.GCS.target}, secret=secret) - self._test_client(call_get_automl_access_token, - '/requests/GetUserSecretRequest', {'Target': GcpTarget.AUTOML.target}, - secret=secret) - self._test_client(call_get_translation_access_token, - '/requests/GetUserSecretRequest', {'Target': GcpTarget.TRANSLATION.target}, - secret=secret) - self._test_client(call_get_natural_lang_access_token, - '/requests/GetUserSecretRequest', {'Target': GcpTarget.NATURAL_LANGUAGE.target}, - secret=secret) - self._test_client(call_get_video_intell_access_token, - '/requests/GetUserSecretRequest', {'Target': GcpTarget.VIDEO_INTELLIGENCE.target}, - secret=secret) - self._test_client(call_get_vision_access_token, - '/requests/GetUserSecretRequest', {'Target': GcpTarget.VISION.target}, + self._test_client(call_get_cloudai_access_token, + '/requests/GetUserSecretRequest', {'Target': GcpTarget.CLOUDAI.target}, secret=secret) def test_get_access_token_handles_unsuccessful(self): diff --git a/tests/test_video_intelligence.py b/tests/test_video_intelligence.py index 856aefdf..15267c85 100644 --- a/tests/test_video_intelligence.py +++ b/tests/test_video_intelligence.py @@ -26,7 +26,7 @@ def __init__(self, user_agent): def test_default_credentials(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VIDEO_INTELLIGENCE') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_video_intelligence() client = videointelligence.VideoIntelligenceServiceClient() @@ -38,7 +38,7 @@ def test_user_provided_credentials(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VIDEO_INTELLIGENCE') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_video_intelligence() client = videointelligence.VideoIntelligenceServiceClient(credentials=credentials) @@ -48,7 +48,7 @@ def test_user_provided_credentials(self): def test_monkeypatching_succeed(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_video_intelligence() client = videointelligence.VideoIntelligenceServiceClient.__init__ @@ -57,7 +57,7 @@ def test_monkeypatching_succeed(self): def test_monkeypatching_idempotent(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VIDEO_INTELLIGENCE') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_video_intelligence() client1 = videointelligence.VideoIntelligenceServiceClient.__init__ diff --git a/tests/test_vision.py b/tests/test_vision.py index 0cd0c295..5fc112bf 100644 --- a/tests/test_vision.py +++ b/tests/test_vision.py @@ -26,7 +26,7 @@ def __init__(self, user_agent): def test_default_credentials(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VISION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_vision() client = vision.ImageAnnotatorClient() @@ -38,7 +38,7 @@ def test_user_provided_credentials(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VISION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_vision() client = vision.ImageAnnotatorClient(credentials=credentials) @@ -48,7 +48,7 @@ def test_user_provided_credentials(self): def test_monkeypatching_succeed(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_vision() client = vision.ImageAnnotatorClient.__init__ @@ -57,7 +57,7 @@ def test_monkeypatching_succeed(self): def test_monkeypatching_idempotent(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VISION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_vision() client1 = vision.ImageAnnotatorClient.__init__ From 4d9f7f9fd87efeae2295a38236c074e0a65d3538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Korn=C3=A9l=20Regius?= Date: Mon, 11 Jan 2021 20:18:43 +0000 Subject: [PATCH 4/5] Simplified Cloud AI integration --- patches/kaggle_gcp.py | 53 +++++++++------------------- patches/kaggle_secrets.py | 22 +++--------- tests/test_automl.py | 30 +++++++++++----- tests/test_natural_language.py | 8 ++--- tests/test_tensorflow_credentials.py | 4 +-- tests/test_translation.py | 14 ++++---- tests/test_user_secrets.py | 20 +++-------- tests/test_video_intelligence.py | 8 ++--- tests/test_vision.py | 8 ++--- 9 files changed, 68 insertions(+), 99 deletions(-) diff --git a/patches/kaggle_gcp.py b/patches/kaggle_gcp.py index d084a0b7..ced3555a 100644 --- a/patches/kaggle_gcp.py +++ b/patches/kaggle_gcp.py @@ -42,20 +42,9 @@ def has_bigquery(self): def has_gcs(self): return GcpTarget.GCS in self.integrations - def has_automl(self): - return GcpTarget.AUTOML in self.integrations - - def has_translation(self): - return GcpTarget.TRANSLATION in self.integrations - - def has_natural_language(self): - return GcpTarget.NATURAL_LANGUAGE in self.integrations - - def has_video_intelligence(self): - return GcpTarget.VIDEO_INTELLIGENCE in self.integrations - - def has_vision(self): - return GcpTarget.VISION in self.integrations + def has_cloudai(self): + return GcpTarget.CLOUDAI in self.integrations or \ + GcpTarget.AUTOML in self.integrations class KaggleKernelCredentials(credentials.Credentials): """Custom Credentials used to authenticate using the Kernel's connected OAuth account. @@ -74,16 +63,8 @@ def refresh(self, request): self.token, self.expiry = client.get_bigquery_access_token() elif self.target == GcpTarget.GCS: self.token, self.expiry = client._get_gcs_access_token() - elif self.target == GcpTarget.AUTOML: - self.token, self.expiry = client._get_automl_access_token() - elif self.target == GcpTarget.TRANSLATION: - self.token, self.expiry = client._get_translation_access_token() - elif self.target == GcpTarget.NATURAL_LANGUAGE: - self.token, self.expiry = client._get_natural_language_access_token() - elif self.target == GcpTarget.VIDEO_INTELLIGENCE: - self.token, self.expiry = client._get_video_intelligence_access_token() - elif self.target == GcpTarget.VISION: - self.token, self.expiry = client._get_vision_access_token() + elif self.target == GcpTarget.CLOUDAI: + self.token, self.expiry = client._get_cloudai_access_token() except ConnectionError as e: Log.error(f"Connection error trying to refresh access token: {e}") print("There was a connection error trying to fetch the access token. " @@ -262,12 +243,12 @@ def init_automl(): return from kaggle_gcp import get_integrations - if not get_integrations().has_automl(): + if not get_integrations().has_cloudai(): return from kaggle_secrets import GcpTarget from kaggle_gcp import KaggleKernelCredentials - kaggle_kernel_credentials = KaggleKernelCredentials(target=GcpTarget.AUTOML) + kaggle_kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI) # Patch the 2 GA clients: AutoMlClient and PreditionServiceClient monkeypatch_client(automl.AutoMlClient, kaggle_kernel_credentials) @@ -293,10 +274,10 @@ def init_translation_v2(): return translate_v2 from kaggle_gcp import get_integrations - if not get_integrations().has_translation(): + if not get_integrations().has_cloudai(): return translate_v2 from kaggle_secrets import GcpTarget - kernel_credentials = KaggleKernelCredentials(target=GcpTarget.TRANSLATION) + kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI) monkeypatch_client(translate_v2.Client, kernel_credentials) return translate_v2 @@ -307,10 +288,10 @@ def init_translation_v3(): return translate_v3 from kaggle_gcp import get_integrations - if not get_integrations().has_translation(): + if not get_integrations().has_cloudai(): return translate_v3 from kaggle_secrets import GcpTarget - kernel_credentials = KaggleKernelCredentials(target=GcpTarget.TRANSLATION) + kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI) monkeypatch_client(translate_v3.TranslationServiceClient, kernel_credentials) return translate_v3 @@ -320,11 +301,11 @@ def init_natural_language(): return language from kaggle_gcp import get_integrations - if not get_integrations().has_natural_language(): + if not get_integrations().has_cloudai(): return language from kaggle_secrets import GcpTarget - kernel_credentials = KaggleKernelCredentials(target=GcpTarget.NATURAL_LANGUAGE) + kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI) monkeypatch_client(language.LanguageServiceClient, kernel_credentials) monkeypatch_client(language.LanguageServiceAsyncClient, kernel_credentials) return language @@ -335,11 +316,11 @@ def init_video_intelligence(): return videointelligence from kaggle_gcp import get_integrations - if not get_integrations().has_video_intelligence(): + if not get_integrations().has_cloudai(): return videointelligence from kaggle_secrets import GcpTarget - kernel_credentials = KaggleKernelCredentials(target=GcpTarget.VIDEO_INTELLIGENCE) + kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI) monkeypatch_client( videointelligence.VideoIntelligenceServiceClient, kernel_credentials) @@ -354,11 +335,11 @@ def init_vision(): return vision from kaggle_gcp import get_integrations - if not get_integrations().has_vision(): + if not get_integrations().has_cloudai(): return vision from kaggle_secrets import GcpTarget - kernel_credentials = KaggleKernelCredentials(target=GcpTarget.VISION) + kernel_credentials = KaggleKernelCredentials(target=GcpTarget.CLOUDAI) monkeypatch_client(vision.ImageAnnotatorClient, kernel_credentials) monkeypatch_client(vision.ImageAnnotatorAsyncClient, kernel_credentials) return vision diff --git a/patches/kaggle_secrets.py b/patches/kaggle_secrets.py index 0aa7c01e..c1c5dc99 100644 --- a/patches/kaggle_secrets.py +++ b/patches/kaggle_secrets.py @@ -23,11 +23,9 @@ class GcpTarget(Enum): """Enum class to store GCP targets.""" BIGQUERY = (1, "BigQuery") GCS = (2, "Google Cloud Storage") + # Old name, should remove later. AUTOML = (3, "Cloud AutoML") - TRANSLATION = (4, "Cloud Translation") - NATURAL_LANGUAGE = (5, "Cloud Natural Language") - VIDEO_INTELLIGENCE = (6, "Cloud Video Intelligence") - VISION = (7, "Cloud Vision") + CLOUDAI = (3, "Google Cloud AI Platform") def __init__(self, target, service): self._target = target @@ -155,20 +153,8 @@ def _write_gsutil_credentials_file(self, credentials) -> str: def _get_gcs_access_token(self) -> Tuple[str, Optional[datetime]]: return self._get_access_token(GcpTarget.GCS) - def _get_automl_access_token(self) -> Tuple[str, Optional[datetime]]: - return self._get_access_token(GcpTarget.AUTOML) - - def _get_translation_access_token(self) -> Tuple[str, Optional[datetime]]: - return self._get_access_token(GcpTarget.TRANSLATION) - - def _get_natural_language_access_token(self) -> Tuple[str, Optional[datetime]]: - return self._get_access_token(GcpTarget.NATURAL_LANGUAGE) - - def _get_video_intelligence_access_token(self) -> Tuple[str, Optional[datetime]]: - return self._get_access_token(GcpTarget.VIDEO_INTELLIGENCE) - - def _get_vision_access_token(self) -> Tuple[str, Optional[datetime]]: - return self._get_access_token(GcpTarget.VISION) + def _get_cloudai_access_token(self) -> Tuple[str, Optional[datetime]]: + return self._get_access_token(GcpTarget.CLOUDAI) def _get_access_token(self, target: GcpTarget) -> Tuple[str, Optional[datetime]]: request_body = { diff --git a/tests/test_automl.py b/tests/test_automl.py index 37571fe9..ba906a5f 100644 --- a/tests/test_automl.py +++ b/tests/test_automl.py @@ -27,7 +27,7 @@ def test_user_provided_credentials(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_automl() client = automl.AutoMlClient(credentials=credentials) @@ -48,7 +48,7 @@ def test_tables_client_credentials(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_automl() tables_client = automl_v1beta1.TablesClient(credentials=credentials) @@ -58,7 +58,7 @@ def test_tables_client_credentials(self): def test_default_credentials_automl_client(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_automl() automl_client = automl.AutoMlClient() @@ -70,7 +70,7 @@ def test_default_credentials_automl_client(self): def test_default_credentials_automl_v1beta1_client(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_automl() automl_client = automl_v1beta1.AutoMlClient() @@ -82,7 +82,7 @@ def test_default_credentials_automl_v1beta1_client(self): def test_default_credentials_tables_client(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_automl() tables_client = automl_v1beta1.TablesClient() @@ -94,7 +94,7 @@ def test_default_credentials_tables_client(self): def test_default_credentials_prediction_client(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: prediction_client = automl.PredictionServiceClient() self.assertIsNotNone(prediction_client.credentials) @@ -105,7 +105,7 @@ def test_default_credentials_prediction_client(self): def test_default_credentials_prediction_v1beta1_client(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: prediction_client = automl_v1beta1.PredictionServiceClient() self.assertIsNotNone(prediction_client.credentials) @@ -115,9 +115,23 @@ def test_default_credentials_prediction_v1beta1_client(self): def test_monkeypatching_idempotent(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: client1 = automl.AutoMlClient.__init__ init_automl() client2 = automl.AutoMlClient.__init__ self.assertEqual(client1, client2) + + @patch("google.cloud.automl_v1beta1.PredictionServiceClient", new=FakeClient) + def test_legacy_AUTOML_variable_v1beta1_client(self): + """ + Tests previous KAGGLE_KERNEL_INTEGRATIONS="AUTOML" environment setting + """ + env = EnvironmentVarGuard() + env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + with env: + prediction_client = automl_v1beta1.PredictionServiceClient() + self.assertIsNotNone(prediction_client.credentials) + self.assertIsInstance(prediction_client.credentials, KaggleKernelCredentials) + self.assertTrue(prediction_client._connection.user_agent.startswith("kaggle-gcp-client/1.0")) \ No newline at end of file diff --git a/tests/test_natural_language.py b/tests/test_natural_language.py index 0ae6f6ed..e2939013 100644 --- a/tests/test_natural_language.py +++ b/tests/test_natural_language.py @@ -26,7 +26,7 @@ def __init__(self, user_agent): def test_default_credentials(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_natural_language() client = language.LanguageServiceClient() @@ -38,7 +38,7 @@ def test_user_provided_credentials(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_natural_language() client = language.LanguageServiceClient(credentials=credentials) @@ -49,7 +49,7 @@ def test_user_provided_credentials(self): def test_monkeypatching_succeed(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_natural_language() client = language.LanguageServiceClient.__init__ @@ -58,7 +58,7 @@ def test_monkeypatching_succeed(self): def test_monkeypatching_idempotent(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'NATURAL_LANGUAGE') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_natural_language() client1 = language.LanguageServiceClient.__init__ diff --git a/tests/test_tensorflow_credentials.py b/tests/test_tensorflow_credentials.py index 42280446..628b35c1 100644 --- a/tests/test_tensorflow_credentials.py +++ b/tests/test_tensorflow_credentials.py @@ -13,7 +13,7 @@ def test_set_tensorflow_credential(self, mock_configure_gcs): credential = '{"client_id":"fake_client_id",' \ '"client_secret":"fake_client_secret",' \ '"refresh_token":"not a refresh token",' \ - '"type":"authorized_user"}'; + '"type":"authorized_user"}' env = EnvironmentVarGuard() env.set('HOME', '/tmp') @@ -22,7 +22,7 @@ def test_set_tensorflow_credential(self, mock_configure_gcs): # These need to be set to make UserSecretsClient happy, but aren't # pertinent to this test. env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'AUTOML') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') user_secrets = UserSecretsClient() user_secrets.set_tensorflow_credential(credential) diff --git a/tests/test_translation.py b/tests/test_translation.py index 83618a14..a55132c9 100644 --- a/tests/test_translation.py +++ b/tests/test_translation.py @@ -28,7 +28,7 @@ def __init__(self, user_agent): def test_default_credentials_v2(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_translation_v2() client = translate_v2.Client() @@ -41,7 +41,7 @@ def test_user_provided_credentials_v2(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_translation_v2() client = translate_v2.Client(credentials=credentials) @@ -52,7 +52,7 @@ def test_user_provided_credentials_v2(self): def test_default_credentials_v3(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_translation_v3() client = translate.TranslationServiceClient() @@ -65,7 +65,7 @@ def test_user_provided_credentials_v3(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_translation_v3() client = translate.TranslationServiceClient(credentials=credentials) @@ -76,7 +76,7 @@ def test_user_provided_credentials_v3(self): def test_monkeypatching_succeed(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_translation_v2() init_translation_v3() @@ -90,7 +90,7 @@ def test_monkeypatching_succeed(self): def test_monkeypatching_idempotent(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_translation_v2() init_translation_v3() @@ -116,7 +116,7 @@ def test_client_credential_uniqueness_v3(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_translation_v3() client1 = translate.TranslationServiceClient() diff --git a/tests/test_user_secrets.py b/tests/test_user_secrets.py index 7f71ccf4..606a5c8c 100644 --- a/tests/test_user_secrets.py +++ b/tests/test_user_secrets.py @@ -178,9 +178,9 @@ def call_get_gcs_access_token(): client = UserSecretsClient() secret_response = client._get_gcs_access_token() self.assertEqual(secret_response, (secret, now + timedelta(seconds=3600))) - def call_get_automl_access_token(): + def call_get_cloudai_access_token(): client = UserSecretsClient() - secret_response = client._get_automl_access_token() + secret_response = client._get_cloudai_access_token() self.assertEqual(secret_response, (secret, now + timedelta(seconds=3600))) def call_get_translation_access_token(): client = UserSecretsClient() @@ -205,20 +205,8 @@ def call_get_vision_access_token(): self._test_client(call_get_gcs_access_token, '/requests/GetUserSecretRequest', {'Target': GcpTarget.GCS.target}, secret=secret) - self._test_client(call_get_automl_access_token, - '/requests/GetUserSecretRequest', {'Target': GcpTarget.AUTOML.target}, - secret=secret) - self._test_client(call_get_translation_access_token, - '/requests/GetUserSecretRequest', {'Target': GcpTarget.TRANSLATION.target}, - secret=secret) - self._test_client(call_get_natural_lang_access_token, - '/requests/GetUserSecretRequest', {'Target': GcpTarget.NATURAL_LANGUAGE.target}, - secret=secret) - self._test_client(call_get_video_intell_access_token, - '/requests/GetUserSecretRequest', {'Target': GcpTarget.VIDEO_INTELLIGENCE.target}, - secret=secret) - self._test_client(call_get_vision_access_token, - '/requests/GetUserSecretRequest', {'Target': GcpTarget.VISION.target}, + self._test_client(call_get_cloudai_access_token, + '/requests/GetUserSecretRequest', {'Target': GcpTarget.CLOUDAI.target}, secret=secret) def test_get_access_token_handles_unsuccessful(self): diff --git a/tests/test_video_intelligence.py b/tests/test_video_intelligence.py index 856aefdf..15267c85 100644 --- a/tests/test_video_intelligence.py +++ b/tests/test_video_intelligence.py @@ -26,7 +26,7 @@ def __init__(self, user_agent): def test_default_credentials(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VIDEO_INTELLIGENCE') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_video_intelligence() client = videointelligence.VideoIntelligenceServiceClient() @@ -38,7 +38,7 @@ def test_user_provided_credentials(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VIDEO_INTELLIGENCE') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_video_intelligence() client = videointelligence.VideoIntelligenceServiceClient(credentials=credentials) @@ -48,7 +48,7 @@ def test_user_provided_credentials(self): def test_monkeypatching_succeed(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_video_intelligence() client = videointelligence.VideoIntelligenceServiceClient.__init__ @@ -57,7 +57,7 @@ def test_monkeypatching_succeed(self): def test_monkeypatching_idempotent(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VIDEO_INTELLIGENCE') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_video_intelligence() client1 = videointelligence.VideoIntelligenceServiceClient.__init__ diff --git a/tests/test_vision.py b/tests/test_vision.py index 0cd0c295..5fc112bf 100644 --- a/tests/test_vision.py +++ b/tests/test_vision.py @@ -26,7 +26,7 @@ def __init__(self, user_agent): def test_default_credentials(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VISION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_vision() client = vision.ImageAnnotatorClient() @@ -38,7 +38,7 @@ def test_user_provided_credentials(self): credentials = _make_credentials() env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VISION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_vision() client = vision.ImageAnnotatorClient(credentials=credentials) @@ -48,7 +48,7 @@ def test_user_provided_credentials(self): def test_monkeypatching_succeed(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'TRANSLATION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_vision() client = vision.ImageAnnotatorClient.__init__ @@ -57,7 +57,7 @@ def test_monkeypatching_succeed(self): def test_monkeypatching_idempotent(self): env = EnvironmentVarGuard() env.set('KAGGLE_USER_SECRETS_TOKEN', 'foobar') - env.set('KAGGLE_KERNEL_INTEGRATIONS', 'VISION') + env.set('KAGGLE_KERNEL_INTEGRATIONS', 'CLOUDAI') with env: init_vision() client1 = vision.ImageAnnotatorClient.__init__ From 8bdc39f1eab59f95b9754c4d2244213a2d428a14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Korn=C3=A9l=20Regius?= Date: Tue, 12 Jan 2021 16:22:43 +0000 Subject: [PATCH 5/5] Fix merge issue in dev.Dockerfile --- dev.Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dev.Dockerfile b/dev.Dockerfile index 7772d621..4a661136 100644 --- a/dev.Dockerfile +++ b/dev.Dockerfile @@ -13,9 +13,8 @@ # # you can run the tests against this new image using: # ./test -i kaggle/python-dev -p test_user_secrets.py # +FROM gcr.io/kaggle-images/python:staging -#FROM gcr.io/kaggle-images/python:staging -FROM kaggle/python-build:latest ADD patches/kaggle_gcp.py /root/.local/lib/python3.7/site-packages/kaggle_gcp.py ADD patches/kaggle_secrets.py /root/.local/lib/python3.7/site-packages/kaggle_secrets.py ADD patches/kaggle_session.py /root/.local/lib/python3.7/site-packages/kaggle_session.py