diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 5daceca..9a76940 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -25,7 +25,7 @@ jobs: virtualenvs-in-project: true - name: Install dependencies - run: poetry install -v -E toml -E yaml -E azure -E aws -E gcp + run: poetry install -v -E toml -E yaml -E azure -E aws -E gcp -E vault - name: Run pytest run: poetry run pytest --cov=./ --cov-report=xml diff --git a/config/contrib/aws.py b/config/contrib/aws.py index bb7bc3d..1cc0388 100644 --- a/config/contrib/aws.py +++ b/config/contrib/aws.py @@ -1,4 +1,4 @@ -"""Configuration from AWS Secrets Manager.""" +"""Configuration instances from AWS Secrets Manager.""" import json import time diff --git a/config/contrib/azure.py b/config/contrib/azure.py index 4f2aaa9..a20d4b7 100644 --- a/config/contrib/azure.py +++ b/config/contrib/azure.py @@ -1,4 +1,4 @@ -"""Configuration from Azure KeyVaults.""" +"""Configuration instances from Azure KeyVaults.""" import time from typing import Any, Dict, ItemsView, KeysView, Optional, Union, ValuesView, cast @@ -88,7 +88,7 @@ def __getitem__(self, item: str) -> Any: # noqa: D105 def __getattr__(self, item: str) -> Any: # noqa: D105 secret = self._get_secret(item) if secret is None: - raise KeyError(item) + raise AttributeError(item) else: return secret diff --git a/config/contrib/gcp.py b/config/contrib/gcp.py index fa63180..df69d20 100644 --- a/config/contrib/gcp.py +++ b/config/contrib/gcp.py @@ -1,4 +1,4 @@ -"""Configuration from GCP Secret Manager.""" +"""Configuration instances from GCP Secret Manager.""" import time from typing import Any, Dict, ItemsView, KeysView, Optional, Union, ValuesView, cast @@ -89,7 +89,7 @@ def __getitem__(self, item: str) -> Any: # noqa: D105 def __getattr__(self, item: str) -> Any: # noqa: D105 secret = self._get_secret(item) if secret is None: - raise KeyError(item) + raise AttributeError(item) else: return secret diff --git a/config/contrib/vault.py b/config/contrib/vault.py new file mode 100644 index 0000000..0e9db11 --- /dev/null +++ b/config/contrib/vault.py @@ -0,0 +1,157 @@ +"""Configuration instances from Hashicorp Vault.""" + +import time +from typing import ( + Any, + Dict, + ItemsView, + KeysView, + Mapping, + Optional, + Union, + ValuesView, + cast, +) + +import hvac +from hvac.exceptions import InvalidPath + + +from .. import Configuration, InterpolateType, config_from_dict + + +class Cache: + """Cache class.""" + + def __init__(self, value: Dict[str, Any], ts: float): # noqa: D107 + self.value = value + self.ts = ts + + +class HashicorpVaultConfiguration(Configuration): + """ + Hashicorp Vault Configuration class. + + The Hashicorp Vault Configuration class takes Vault credentials and + behaves like a drop-in replacement for the regular Configuration class. + + The following limitations apply to the Hashicorp Vault Configurations: + - only works with KV version 2 + - only supports the latest secret version + - assumes that secrets are named as // + """ + + def __init__( + self, + engine: str, + cache_expiration: int = 5 * 60, + interpolate: InterpolateType = False, + **kwargs: Mapping[str, Any], + ) -> None: + """ + Constructor. + + See https://developer.hashicorp.com/vault/docs/get-started/developer-qs. + """ # noqa: E501 + self._client = hvac.Client(**kwargs) + self._cache_expiration = cache_expiration + self._cache: Dict[str, Cache] = {} + self._engine = engine + self._interpolate = {} if interpolate is True else interpolate + self._default_levels = None + + def _get_secret(self, secret: str) -> Optional[Dict[str, Any]]: + now = time.time() + from_cache = self._cache.get(secret) + if from_cache and from_cache.ts + self._cache_expiration > now: + return from_cache.value + try: + data = cast( + Dict[str, Any], + self._client.kv.v2.read_secret(secret, mount_point=self._engine)[ + "data" + ]["data"], + ) + self._cache[secret] = Cache(value=data, ts=now) + return data + except (InvalidPath, KeyError): + if secret in self._cache: + del self._cache[secret] + return None + + def __getitem__(self, item: str) -> Any: # noqa: D105 + path, *rest = item.split(".", 1) + secret = self._get_secret(path) + if secret is None: + raise KeyError(item) + else: + return ( + Configuration(secret)[".".join(rest)] if rest else Configuration(secret) + ) + + def __getattr__(self, item: str) -> Any: # noqa: D105 + secret = self._get_secret(item) + if secret is None: + raise AttributeError(item) + else: + return Configuration(secret) + + def get(self, key: str, default: Any = None) -> Union[dict, Any]: + """ + Get the configuration values corresponding to :attr:`key`. + + :param key: key to retrieve + :param default: default value in case the key is missing + :return: the value found or a default + """ + try: + return self[key] + except KeyError: + return default + + def keys( + self, levels: Optional[int] = None + ) -> Union["Configuration", Any, KeysView[str]]: + """Return a set-like object providing a view on the configuration keys.""" + assert not levels # Vault secrets don't support separators + return cast( + KeysView[str], + self._client.list(f"/{self._engine}/metadata")["data"]["keys"], + ) + + def values( + self, levels: Optional[int] = None + ) -> Union["Configuration", Any, ValuesView[Any]]: + """Return a set-like object providing a view on the configuration values.""" + assert not levels # GCP Secret Manager secrets don't support separators + return cast( + ValuesView[str], + ( + self._get_secret(k) + for k in self._client.list(f"/{self._engine}/metadata")["data"]["keys"] + ), + ) + + def items( + self, levels: Optional[int] = None + ) -> Union["Configuration", Any, ItemsView[str, Any]]: + """Return a set-like object providing a view on the configuration items.""" + assert not levels # GCP Secret Manager secrets don't support separators + return cast( + ItemsView[str, Any], + ( + (k, self._get_secret(k)) + for k in self._client.list(f"/{self._engine}/metadata")["data"]["keys"] + ), + ) + + def reload(self) -> None: + """Reload the configuration.""" + self._cache.clear() + + def __repr__(self) -> str: # noqa: D105 + return "" % self._engine + + @property + def _config(self) -> Dict[str, Any]: # type: ignore + return config_from_dict(dict(self.items()))._config diff --git a/pyproject.toml b/pyproject.toml index a2cd076..f0ed2ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ azure-identity = { version = "^1.13.0", optional = true } azure-keyvault = { version = "^4.2.0", optional = true } boto3 = { version = "^1.28.20", optional = true } google-cloud-secret-manager = { version = "^2.16.3", optional = true } +hvac = { version ="^1.1.1", optional = true } pyyaml = { version = "^6.0", optional = true } toml = { version = "^0.10.0", optional = true } @@ -44,6 +45,7 @@ aws = ["boto3"] azure = ["azure-keyvault", "azure-identity"] gcp = ["google-cloud-secret-manager"] toml = ["toml"] +vault = ["hvac"] yaml = ["pyyaml"] [tool.black] @@ -58,10 +60,72 @@ envlist = py38, py39, py310, py311 [testenv] allowlist_externals = poetry commands = - poetry install -v -E toml -E yaml -E azure -E aws -E gcp + poetry install -v -E toml -E yaml -E azure -E aws -E gcp -E vault poetry run pytest """ +[tool.mypy] +warn_return_any = true +warn_unused_configs = true +disallow_untyped_calls = true +disallow_untyped_defs = true +disallow_subclassing_any = true +disallow_any_decorated = true +disallow_incomplete_defs = true +disallow_untyped_decorators = true +no_implicit_optional = true +warn_unused_ignores = true +warn_redundant_casts = true +exclude = [ + 'tests' +] + +[[tool.mypy.overrides]] +module= [ + 'google.auth.credentials', + 'yaml', + 'toml', + 'boto3', + 'botocore.exceptions', + 'hvac', + 'hvac.exceptions', +] +ignore_missing_imports = true + +[tool.coverage.run] +branch = true +include = [ + 'config/*' +] + +[tool.coverage.html] +directory = 'cover' + +[tool.pytest.ini_options] +minversion = "6.0" +addopts = '--cov --cov-report=html --cov-report term-missing --flake8 --mypy --black' +flake8-max-line-length = 88 +flake8-extensions =[ + 'flake8-docstrings', + 'flake8-comprehensions', + 'flake8-import-order', + 'flake8-bugbear', + 'flake8-blind-except', + 'flake8-builtins', + 'flake8-logging-format', + 'flake8-black' +] +flake8-ignore = [ + '* E203', + 'tests/* ALL', + 'docs/* ALL' + ] +filterwarnings =[ + 'ignore::pytest.PytestDeprecationWarning', + 'ignore::DeprecationWarning', + 'ignore::pytest.PytestWarning' +] + [build-system] build-backend = "poetry.masonry.api" requires = ["poetry>=1.5.0"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index ed6b700..0000000 --- a/setup.cfg +++ /dev/null @@ -1,70 +0,0 @@ -[tool:pytest] -addopts = --cov --cov-report=html --cov-report term-missing --flake8 --mypy --black -flake8-max-line-length = 88 -flake8-extensions = - flake8-docstrings - flake8-comprehensions - flake8-import-order - flake8-bugbear - flake8-blind-except - flake8-builtins - flake8-logging-format - flake8-black -flake8-ignore = - * E203 - tests/* ALL - docs/* ALL -filterwarnings = - ignore::pytest.PytestDeprecationWarning - ignore::DeprecationWarning - ignore::pytest.PytestWarning - -[aliases] -test=pytest - -[coverage:run] -branch = True -include = config/* - -[coverage:html] -directory = cover - -[mypy] -warn_return_any = True -warn_unused_configs = True -disallow_untyped_calls = True -disallow_untyped_defs = True -disallow_subclassing_any = True -disallow_any_decorated = True -disallow_incomplete_defs = True -disallow_untyped_decorators = True -no_implicit_optional = True -warn_unused_ignores = True -warn_redundant_casts = True - -[mypy-pytest.*] -ignore_missing_imports = True - -[mypy-azure.*] -ignore_missing_imports = True - -[mypy-boto3.*] -ignore_missing_imports = True - -[mypy-botocore.*] -ignore_missing_imports = True - -[mypy-google.*] -ignore_missing_imports = True - -[mypy-yaml.*] -ignore_missing_imports = True - -[mypy-toml.*] -ignore_missing_imports = True - -[flake8] -exclude = - .* -max-line-length = 88 - diff --git a/tests/contrib/test_azure.py b/tests/contrib/test_azure.py index 4375c92..5a62f5c 100644 --- a/tests/contrib/test_azure.py +++ b/tests/contrib/test_azure.py @@ -124,8 +124,8 @@ def test_get_attr(): # type: ignore assert cfg.foo == "foo_val" - with raises(KeyError): - assert cfg.foo_missing is KeyError + with raises(AttributeError): + assert cfg.foo_missing is AttributeError @pytest.mark.skipif("azure is None") diff --git a/tests/contrib/test_gcp.py b/tests/contrib/test_gcp.py index 6c7e103..3d4bbf1 100644 --- a/tests/contrib/test_gcp.py +++ b/tests/contrib/test_gcp.py @@ -57,9 +57,6 @@ def call(*args: list, **kwargs: dict) -> FakeSecretClient: def test_load_dict(): # type: ignore secretmanager_v1.SecretManagerServiceClient = fake_client(DICT) cfg = GCPSecretManagerConfiguration("fake_id") - - print(cfg) - assert cfg["foo"] == "foo_val" assert cfg["with_underscore"] == "works" assert cfg.get("foo", "default") == "foo_val" @@ -120,8 +117,8 @@ def test_get_attr(): # type: ignore assert cfg.foo == "foo_val" - with raises(KeyError): - assert cfg.foo_missing is KeyError + with raises(AttributeError): + assert cfg.foo_missing is AttributeError @pytest.mark.skipif("secretmanager_v1 is None") diff --git a/tests/contrib/test_vault.py b/tests/contrib/test_vault.py new file mode 100644 index 0000000..75ca7c5 --- /dev/null +++ b/tests/contrib/test_vault.py @@ -0,0 +1,167 @@ +from collections import namedtuple +import pytest +from pytest import raises + +from config import config_from_dict + +try: + import hvac + from config.contrib.vault import HashicorpVaultConfiguration +except ImportError: # pragma: no cover + hvac = None + +DICT = { + "foo": "foo_val", + "bar": "bar_val", + "with_underscore": "works", + "password": "some passwd", +} + +DICT2 = {"a": "b", "c": "d"} + +FakeKeySecret = namedtuple("FakeKeySecret", ["key", "value"]) + + +class FakeSecretClient: + def __init__(self, engine, dct: dict): # type: ignore + self._engine = engine + self._dict = dct + + @property + def kv(self): # type: ignore + return self + + @property + def v2(self): # type: ignore + return self + + def read_secret(self, secret, mount_point): # type: ignore + if mount_point == self._engine: + return {"data": {"data": config_from_dict(self._dict[secret]).as_dict()}} + else: + raise KeyError + + def list(self, path): # type: ignore + return {"data": {"keys": list(self._dict.keys())}} + + +@pytest.mark.skipif("hvac is None") +def test_load_dict(): # type: ignore + cfg = HashicorpVaultConfiguration("engine") + cfg._client = FakeSecretClient("engine", {"k": DICT}) + + assert cfg["k"]["foo"] == "foo_val" + assert cfg["k"]["with_underscore"] == "works" + assert cfg.get("k.foo", "default") == "foo_val" + + +@pytest.mark.skipif("hvac is None") +def test_expiration(mocker): # type: ignore + # with cache + cfg = HashicorpVaultConfiguration("engine") + cfg._client = FakeSecretClient("engine", {"k": DICT}) + + spy = mocker.spy(cfg._client, "read_secret") + assert cfg["k"]["foo"] == "foo_val" + assert cfg["k"]["foo"] == "foo_val" + assert spy.call_count == 1 + + # without cache + cfg = HashicorpVaultConfiguration("engine", cache_expiration=0) + cfg._client = FakeSecretClient("engine", {"k": DICT}) + + spy = mocker.spy(cfg._client, "read_secret") + assert cfg["k"]["foo"] == "foo_val" + assert cfg["k"]["foo"] == "foo_val" # this will ignore the cache + assert spy.call_count == 2 + + +@pytest.mark.skipif("hvac is None") +def test_deletion(): # type: ignore + cfg = HashicorpVaultConfiguration("engine", cache_expiration=0) + d = DICT.copy() + dd = {"k": d, "a": d} + cfg._client = FakeSecretClient("engine", dd) + + assert cfg.k["foo"] == "foo_val" + assert "k" in cfg._cache + del dd["k"] + + with raises(KeyError): + assert cfg["k"] is KeyError + + +@pytest.mark.skipif("hvac is None") +def test_missing_key(): # type: ignore + cfg = HashicorpVaultConfiguration("engine") + d = DICT.copy() + cfg._client = FakeSecretClient("engine", {"k": d}) + + with raises(KeyError): + assert cfg["not-k"] is KeyError + + with raises(KeyError): + assert cfg["k"]["foo-missing"] is KeyError + + assert cfg.get("k.foo-missing", "default") == "default" + + +@pytest.mark.skipif("hvac is None") +def test_get_attr(): # type: ignore + cfg = HashicorpVaultConfiguration("engine") + d = DICT.copy() + cfg._client = FakeSecretClient("engine", {"k": d}) + + assert cfg.k.foo == "foo_val" + + with raises(AttributeError): + assert cfg.notk is AttributeError + + with raises(AttributeError): + assert cfg.k.foo_missing is AttributeError + + +@pytest.mark.skipif("hvac is None") +def test_dict(): # type: ignore + cfg = HashicorpVaultConfiguration("engine") + d = DICT.copy() + cfg._client = FakeSecretClient("engine", {"k": d, "a": d}) + + assert sorted(cfg.keys()) == sorted({"k": d, "a": d}.keys()) + assert list(cfg.values()) == [d, d] + assert sorted(cfg.items()) == sorted({"k": d, "a": d}.items()) + + +@pytest.mark.skipif("hvac is None") +def test_repr(): # type: ignore + cfg = HashicorpVaultConfiguration("engine") + d = DICT.copy() + cfg._client = FakeSecretClient("engine", {"k": d}) + + assert repr(cfg) == "" + + +@pytest.mark.skipif("hvac is None") +def test_str(): # type: ignore + cfg = HashicorpVaultConfiguration("engine") + d = DICT.copy() + cfg._client = FakeSecretClient("engine", {"k": d}) + + # str + assert ( + str(cfg) + == "{'k.bar': 'bar_val', 'k.foo': 'foo_val', 'k.password': '******', 'k.with_underscore': 'works'}" + ) + assert cfg["k.password"] == "some passwd" + + +@pytest.mark.skipif("hvac is None") +def test_reload(): # type: ignore + cfg = HashicorpVaultConfiguration("engine") + d = DICT.copy() + cfg._client = FakeSecretClient("engine", {"k": d}) + assert cfg == config_from_dict({"k": DICT}) + + cfg._client = FakeSecretClient("engine", {"k": DICT2}) + cfg.reload() + assert cfg == config_from_dict({"k": DICT2})