New develop merger (#645)

Bring mli-feature up to date with develop.

---------

[ committed by @al-rigazzi ]
[ reviewed by @AlyssaCote @ankona ]

Author: al-rigazzi

.readthedocs.yaml

@@ -23,7 +23,7 @@ build:
       - git clone --depth 1 https://github.com/CrayLabs/SmartRedis.git smartredis
       - git clone --depth 1 https://github.com/CrayLabs/SmartDashboard.git smartdashboard
     post_create_environment:
-      - python -m pip install .[dev]
+      - python -m pip install .[dev,docs]
       - cd smartredis; python -m pip install .
       - cd smartredis/doc; doxygen Doxyfile_c; doxygen Doxyfile_cpp; doxygen Doxyfile_fortran
       - ln -s smartredis/examples ./examples
@@ -37,7 +37,3 @@ build:
 sphinx:
   configuration: doc/conf.py
   fail_on_warning: true
-
-python:
-   install:
-   - requirements: doc/requirements-doc.txt

doc/changelog.md

@@ -30,6 +30,8 @@ To be released at some future point in time
 
 Description
 
+- Fix internal host name representation for Dragon backend
+- Make dependencies more discoverable in setup.py
 - Add hardware pinning capability when using dragon
 - Pin NumPy version to 1.x
 - New launcher support for SGE (and similar derivatives)
@@ -40,6 +42,14 @@ Description
 
 Detailed Notes
 
+- setup.py used to define dependencies in a way that was not amenable
+  to code scanning tools. Direct dependencies now appear directly
+  in the setup call and the definition of the SmartRedis version
+  has been removed
+  ([SmartSim-PR635](https://github.com/CrayLabs/SmartSim/pull/635))
+- The separate definition of dependencies for the docs in
+  requirements-doc.txt is now defined as an extra.
+  ([SmartSim-PR635](https://github.com/CrayLabs/SmartSim/pull/635))
 - The new major version release of Numpy is incompatible with modules
   compiled against Numpy 1.x. For both SmartSim and SmartRedis we
   request a 1.x version of numpy. This is needed in SmartSim because

doc/requirements-doc.txt

@@ -55,8 +55,7 @@ RUN git clone https://github.com/CrayLabs/SmartDashboard.git --branch develop --
     && rm -rf ~/.cache/pip
 
 # Install docs dependencies and SmartSim
-RUN python -m pip install -r doc/requirements-doc.txt \
-    && NO_CHECKS=1 SMARTSIM_SUFFIX=dev python -m pip install .
+RUN NO_CHECKS=1 SMARTSIM_SUFFIX=dev python -m pip install .[docs]
 
 # Note this is needed to ensure that the Sphinx builds. Can be removed with newer Tensorflow
 RUN python -m pip install typing_extensions==4.6.1

docker/docs/dev/Dockerfile

@@ -165,26 +165,9 @@ def has_ext_modules(_placeholder):
 
 
 # Define needed dependencies for the installation
-deps = [
-    "packaging>=24.0",
-    "psutil>=5.7.2",
-    "coloredlogs>=10.0",
-    "tabulate>=0.8.9",
-    "redis>=4.5",
-    "tqdm>=4.50.2",
-    "filelock>=3.4.2",
-    "protobuf~=3.20",
-    "jinja2>=3.1.2",
-    "watchdog>=4.0.0",
-    "pycapnp==2.0.0",
-    "pydantic==1.10.14",
-    "pyzmq>=25.1.2",
-    "pygithub>=2.3.0",
-    "numpy<2"
-]
 
 # Add SmartRedis at specific version
-deps.append("smartredis>={}".format(versions.SMARTREDIS))
+# install_requires.append("smartredis>={}".format(versions.SMARTREDIS))
 
 extras_require = {
     "dev": [
@@ -206,6 +189,24 @@ def has_ext_modules(_placeholder):
         "types-setuptools",
         "typing_extensions>=4.1.0",
     ],
+    "docs": [
+        "Sphinx==6.2.1",
+        "breathe==4.35.0",
+        "sphinx-fortran==1.1.1",
+        "sphinx-book-theme==1.0.1",
+        "sphinx-copybutton==0.5.2",
+        "sphinx-tabs==3.4.4",
+        "nbsphinx==0.9.3",
+        "docutils==0.18.1",
+        "torch==2.0.1",
+        "tensorflow==2.13.1",
+        "ipython",
+        "jinja2==3.1.2",
+        "sphinx-design",
+        "pypandoc",
+        "sphinx-autodoc-typehints",
+        "myst_parser",
+    ],
     # see smartsim/_core/_install/buildenv.py for more details
     **versions.ml_extras_required(),
 }
@@ -214,7 +215,24 @@ def has_ext_modules(_placeholder):
 # rest in setup.cfg
 setup(
     version=smartsim_version,
-    install_requires=deps,
+    install_requires=[
+        "packaging>=24.0",
+        "psutil>=5.7.2",
+        "coloredlogs>=10.0",
+        "tabulate>=0.8.9",
+        "redis>=4.5",
+        "tqdm>=4.50.2",
+        "filelock>=3.4.2",
+        "protobuf~=3.20",
+        "jinja2>=3.1.2",
+        "watchdog>=4.0.0",
+        "pycapnp==2.0.0",
+        "pydantic==1.10.14",
+        "pyzmq>=25.1.2",
+        "pygithub>=2.3.0",
+        "numpy<2",
+        "smartredis>=0.5,<0.6",
+    ],
     cmdclass={
         "build_py": SmartSimBuild,
         "install": InstallPlatlib,

setup.py

@@ -242,7 +242,7 @@ class Versioner:
     ``smart build`` command to determine which dependency versions
     to look for and download.
 
-    Default versions for SmartSim, SmartRedis, Redis, and RedisAI are
+    Default versions for SmartSim, Redis, and RedisAI are
     all set here. Setting a default version for RedisAI also dictates
     default versions of the machine learning libraries.
     """
@@ -252,7 +252,6 @@ class Versioner:
 
     # Versions
     SMARTSIM = Version_(get_env("SMARTSIM_VERSION", "0.7.0"))
-    SMARTREDIS = Version_(get_env("SMARTREDIS_VERSION", "0.5.3"))
     SMARTSIM_SUFFIX = get_env("SMARTSIM_SUFFIX", "")
 
     # Redis
@@ -284,7 +283,6 @@ class Versioner:
     def as_dict(self, db_name: DbEngine = "REDIS") -> t.Dict[str, t.Tuple[str, ...]]:
         pkg_map = {
             "SMARTSIM": self.SMARTSIM,
-            "SMARTREDIS": self.SMARTREDIS,
             db_name: self.REDIS,
             "REDISAI": self.REDISAI,
             "TORCH": self.TORCH,

smartsim/_core/_install/buildenv.py

@@ -213,10 +213,10 @@ def group_infos(self) -> dict[str, ProcessGroupInfo]:
 
     def _initialize_hosts(self) -> None:
         with self._queue_lock:
-            self._hosts: t.List[str] = sorted(
-                node for node in dragon_machine.System().nodes
-            )
-            self._nodes = [dragon_machine.Node(node) for node in self._hosts]
+            self._nodes = [
+                dragon_machine.Node(node) for node in dragon_machine.System().nodes
+            ]
+            self._hosts: t.List[str] = sorted(node.hostname for node in self._nodes)
             self._cpus = [node.num_cpus for node in self._nodes]
             self._gpus = [node.num_gpus for node in self._nodes]
 
@@ -471,7 +471,11 @@ def create_run_policy(
                 if run_request.policy.gpu_affinity:
                     affinity = dragon_policy.Policy.Affinity.SPECIFIC
                     gpu_affinity = run_request.policy.gpu_affinity
-
+            logger.debug(
+                f"Affinity strategy: {affinity}, "
+                f"CPU affinity mask: {cpu_affinity}, "
+                f"GPU affinity mask: {gpu_affinity}"
+            )
             if affinity != dragon_policy.Policy.Affinity.DEFAULT:
                 return dragon_policy.Policy(
                     placement=dragon_policy.Policy.Placement.HOST_NAME,