diff --git a/Dockerfile b/Dockerfile index e3877a6ecf..d220b45a73 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,8 +22,8 @@ RUN python3 -m pip install --no-cache-dir /tmp/suzieq-$version-py3-none-any.whl rm -rf /tmp/* /var/tmp/* VOLUME [ "/home/$username/parquet" ] - -ENV PATH=/root/.local/bin:$PATH:/root/.local/lib/python3.7/site-packages/suzieq/cli/:/root/.local/lib/python3.7/site-packages/suzieq/poller/:/root/.local/lib/python3.7/site-packages/suzieq/restServer + +ENV PATH=/root/.local/bin:$PATH:/root/.local/lib/python3.9/site-packages/suzieq/cli/:/root/.local/lib/python3.9/site-packages/suzieq/poller/:/root/.local/lib/python3.9/site-packages/suzieq/restServer ENV SQENV=docker diff --git a/Dockerfile-sqbase b/Dockerfile-sqbase index 45072af4cc..745a4b456d 100644 --- a/Dockerfile-sqbase +++ b/Dockerfile-sqbase @@ -1,4 +1,4 @@ -FROM python:3.8.14-slim-buster AS sqbase +FROM python:3.9.21-slim-bullseye AS sqbase ENV PATH=/root/.local/bin:$PATH ENV PYTHONPATH=/root/.local/lib diff --git a/build/requirements.txt b/build/requirements.txt index 4da43b63d9..ece2e75a4a 100644 --- a/build/requirements.txt +++ b/build/requirements.txt @@ -111,6 +111,7 @@ python-dateutil==2.8.2 ; python_full_version > "3.8.1" and python_version < "3.1 python-dotenv==0.19.2 ; python_full_version > "3.8.1" and python_version < "3.10" python-nubia==0.2b5 ; python_full_version > "3.8.1" and python_version < "3.10" pytz==2023.3 ; python_full_version > "3.8.1" and python_version < "3.10" +pyvmomi==8.0.2.0.1 ; python_full_version > "3.8.1" and python_version < "3.10" pywin32==306 ; sys_platform == "win32" and platform_python_implementation != "PyPy" and python_full_version > "3.8.1" and python_version < "3.10" pywinpty==2.0.11 ; python_full_version > "3.8.1" and python_version < "3.10" and os_name == "nt" pyyaml==6.0 ; python_full_version > "3.8.1" and python_version < "3.10" @@ -145,7 +146,7 @@ typing-extensions==4.7.1 ; python_full_version > "3.8.1" and python_version < "3 typing-inspect==0.9.0 ; python_full_version > "3.8.1" and python_version < "3.10" tzlocal==2.1 ; python_full_version > "3.8.1" and python_version < "3.10" urllib3==1.26.16 ; python_full_version > "3.8.1" and python_version < "3.10" -uvicorn==0.14.0 ; python_full_version > "3.8.1" and python_version < "3.10" +uvicorn==0.15.0 ; python_full_version > "3.8.1" and python_version < "3.10" uvloop==0.17.0 ; python_full_version > "3.8.1" and python_version < "3.10" validators==0.20.0 ; python_full_version > "3.8.1" and python_version < "3.10" watchdog==3.0.0 ; python_full_version > "3.8.1" and python_version < "3.10" and platform_system != "Darwin" diff --git a/poetry.lock b/poetry.lock index eb4f2a0c13..14b34a6986 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4727,19 +4727,19 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] [[package]] name = "uvicorn" -version = "0.14.0" +version = "0.15.0" description = "The lightning-fast ASGI server." category = "main" optional = false python-versions = "*" files = [ - {file = "uvicorn-0.14.0-py3-none-any.whl", hash = "sha256:2a76bb359171a504b3d1c853409af3adbfa5cef374a4a59e5881945a97a93eae"}, - {file = "uvicorn-0.14.0.tar.gz", hash = "sha256:45ad7dfaaa7d55cab4cd1e85e03f27e9d60bc067ddc59db52a2b0aeca8870292"}, + {file = "uvicorn-0.15.0-py3-none-any.whl", hash = "sha256:17f898c64c71a2640514d4089da2689e5db1ce5d4086c2d53699bf99513421c1"}, + {file = "uvicorn-0.15.0.tar.gz", hash = "sha256:d9a3c0dd1ca86728d3e235182683b4cf94cd53a867c288eaeca80ee781b2caff"}, ] [package.dependencies] -asgiref = ">=3.3.4" -click = ">=7" +asgiref = ">=3.4.0" +click = ">=7.0" h11 = ">=0.8" [package.extras] @@ -5111,4 +5111,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">3.8.1, < 3.10" -content-hash = "ece1db60a0522861ddeff3f1274a91e08181ac0bafc0955199cbd0a934b6b2f8" +content-hash = "52baef73e079105051f17534c6fd05f66e760560dcf44b6f4541d625687a2bdc" diff --git a/pyproject.toml b/pyproject.toml index a6bd7bad57..07d32a18a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ faker = "~4.18" netconan = "~0.11.2" jsonpath-ng = "~1.6" fastapi = "~0.95.2" -uvicorn = "^0.14.0" +uvicorn = "^0.15.0" tabulate = "^0.8.7" streamlit = "1.11.1" graphviz = "^0.15" diff --git a/suzieq/poller/worker/nodes/node.py b/suzieq/poller/worker/nodes/node.py index 112e17487d..68031c65bb 100644 --- a/suzieq/poller/worker/nodes/node.py +++ b/suzieq/poller/worker/nodes/node.py @@ -12,6 +12,7 @@ from urllib.parse import urlparse import asyncio from asyncio.subprocess import PIPE, DEVNULL +import os # pylint: disable=redefined-builtin from concurrent.futures._base import TimeoutError @@ -141,27 +142,21 @@ async def initialize(self, **kwargs) -> TNode: self.ssh_config_file = kwargs.get("ssh_config_file", None) self.enable_password = kwargs.get('enable_password') - passphrase: str = kwargs.get("passphrase", None) + self.pvtkey_file = kwargs.get("ssh_keyfile", None) + self.jump_host_pvtkey_file = kwargs.get("jump_host_key_file", None) + self.passphrase: str = kwargs.get("passphrase", None) + jump_host = kwargs.get("jump_host", "") if jump_host: jump_result = urlparse(jump_host) self.jump_user = jump_result.username or self.username self.jump_host = jump_result.hostname - self.jump_host_key = None if jump_result.port: self.jump_port = jump_result.port else: self.jump_port = 22 - pvtkey_file = kwargs.pop('jump_host_key_file') - if pvtkey_file: - self.jump_host_key = self._decrypt_pvtkey(pvtkey_file, - passphrase) - if not self.jump_host_key: - raise SqPollerConfError('Unable to read private key file' - f' at {pvtkey_file}') else: self.jump_host = None - self.jump_host_key = None self.ignore_known_hosts = kwargs.get('ignore_known_hosts', False) self.slow_host = kwargs.get('slow_host', False) @@ -173,15 +168,6 @@ async def initialize(self, **kwargs) -> TNode: # 4 is a number we picked to limit using up too many SSH sessions # Many newer implementations allow upto 5 simultaneous SSH sessions self.batch_size = 4 - pvtkey_file = kwargs.get("ssh_keyfile", None) - if pvtkey_file: - self.pvtkey = self._decrypt_pvtkey(pvtkey_file, passphrase) - if not self.pvtkey: - self.logger.error("ERROR: Falling back to password for " - f"{self.address}:{self.port}") - self.pvtkey = None - else: - self.pvtkey = None self._init_service_queue() @@ -257,8 +243,8 @@ def _decrypt_pvtkey(self, pvtkey_file: str, passphrase: str) -> str: passphrase) except Exception as e: # pylint: disable=broad-except self.logger.error( - f"ERROR: Unable to read private key file {pvtkey_file}" - f"for jump host due to {e}") + f"ERROR: Unable to read private key file {pvtkey_file} " + f"due to {e}") return keydata @@ -570,9 +556,10 @@ async def _init_jump_host_connection( if self._tunnel: return - if self.jump_host_key: + if self.jump_host_pvtkey_file: jump_host_options = asyncssh.SSHClientConnectionOptions( - client_keys=self.jump_host_key, + client_keys=self.jump_host_pvtkey_file, + passphrase=self.passphrase, connect_timeout=self.connect_timeout, ) @@ -618,9 +605,7 @@ def _init_ssh_options(self) -> asyncssh.SSHClientConnectionOptions: options = asyncssh.SSHClientConnectionOptions( connect_timeout=self.connect_timeout, username=self.username, - agent_identities=self.pvtkey if self.pvtkey else None, - client_keys=self.pvtkey if self.pvtkey else None, - password=self.password if not self.pvtkey else None, + password=self.password if not self.pvtkey_file else None, kex_algs='+diffie-hellman-group1-sha1', # for older boxes encryption_algs='+aes256-cbc', # for older boxes ) @@ -635,6 +620,22 @@ def _init_ssh_options(self) -> asyncssh.SSHClientConnectionOptions: config=[self.ssh_config_file], ) + if self.pvtkey_file: + # Giving just the filename let's asyncssh know to look for the + # corresponding cert file in the same directory. + # Ref: https://asyncssh.readthedocs.io/en/stable/api.html#specifying-private-keys + client_keys = self.pvtkey_file + options = asyncssh.SSHClientConnectionOptions( + options=options, + client_keys=client_keys, + ) + + if self.passphrase: + options = asyncssh.SSHClientConnectionOptions( + options=options, + passphrase=self.passphrase, + ) + return options async def _init_ssh(self, init_dev_data=True, use_lock=True) -> None: