Pyright

.github/workflows/Lint-and-test.yml

@@ -0,0 +1,42 @@
+name: Lint-and-test
+on: [pull_request, workflow_call]
+jobs:
+  call-linter-workflow:
+    uses: ISISComputingGroup/reusable-workflows/.github/workflows/linters.yml@main
+    with:
+      compare-branch: origin/master
+      python-ver: '3.11'
+  tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ "ubuntu-latest" ]
+        # Wide matrix of versions as this may run on a RHEL node with old python versions,
+        # but we also want it to work on dev machines
+        version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+        include:
+          - os: "windows-latest"
+            version: '3.11'
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.version }}
+      - name: install requirements
+        run: pip install -e .[dev]
+      - name: run pytest (linux)
+        run: python -m pytest
+  results:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    name: Final Results
+    needs: [call-linter-workflow, tests]
+    steps:
+    - run: exit 1
+      # see https://stackoverflow.com/a/67532120/4907315
+      if: >-
+          ${{
+               contains(needs.*.result, 'failure')
+            || contains(needs.*.result, 'cancelled')
+          }}

.gitignore

@@ -17,3 +17,8 @@ relPaths.sh
 *.kdb
 *.kdbx
 /exp_db_populator_venv
+/.venv
+/.coverage
+*.egg-info
+logs
+/build

create_rb_number_populator_python_venv.sh

@@ -3,6 +3,6 @@
 venv="exp_db_populator_venv" # Name of the virtual environment
 /usr/local/bin/python3.8 -m venv /home/epics/RB_num_populator/$venv # create virtual environment
 source $venv/bin/activate # activate the virtual environment
-$venv/bin/pip install -r requirements.txt # Install requirements.txt
+$venv/bin/pip install -e . # Install requirements.txt
 deactivate # deactivate the virtual environment
 echo "Virtual environment created"

main.py → exp_db_populator/cli.py

@@ -3,6 +3,12 @@
 from datetime import datetime
 from logging.handlers import TimedRotatingFileHandler
 
+from exp_db_populator.data_types import InstList, RawDataEntry
+from exp_db_populator.webservices_test_data import (
+    TEST_USER_1,
+    create_web_data_with_experimenters_and_other_date,
+)
+
 # Loging must be handled here as some imports might log errors
 log_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), "logs")
 if not os.path.exists(log_folder):
@@ -21,31 +27,27 @@
 import json
 import threading
 import zlib
+from typing import Any
 
 import epics
-from six.moves import input
 
 from exp_db_populator.gatherer import Gatherer
 from exp_db_populator.populator import update
 from exp_db_populator.webservices_reader import reformat_data
-from tests.webservices_test_data import (
-    TEST_USER_1,
-    create_web_data_with_experimenters_and_other_date,
-)
 
 # PV that contains the instrument list
 INST_LIST_PV = "CS:INSTLIST"
 
 
-def convert_inst_list(value_from_PV):
+def convert_inst_list(value_from_pv: str) -> InstList:
     """
     Converts the instrument list coming from the PV into a dictionary.
     Args:
-        value_from_PV: The raw value from the PV.
+        value_from_pv: The raw value from the PV.
     Returns:
         dict: The instrument information.
     """
-    json_string = zlib.decompress(bytes.fromhex(value_from_PV)).decode("utf-8")
+    json_string = zlib.decompress(bytes.fromhex(value_from_pv)).decode("utf-8")
     return json.loads(json_string)
 
 
@@ -55,30 +57,33 @@ class InstrumentPopulatorRunner:
     """
 
     gatherer = None
-    prev_inst_list = None
+    prev_inst_list: InstList | None = None
     db_lock = threading.RLock()
 
-    def __init__(self, run_continuous=False):
+    def __init__(self, run_continuous: bool = False) -> None:
         self.run_continuous = run_continuous
 
-    def start_inst_list_monitor(self):
+    def start_inst_list_monitor(self) -> None:
         logging.info("Setting up monitors on {}".format(INST_LIST_PV))
         self.inst_list_callback(char_value=epics.caget(INST_LIST_PV, as_string=True))
         epics.camonitor(INST_LIST_PV, callback=self.inst_list_callback)
 
-    def inst_list_callback(self, char_value, **kw):
+    def inst_list_callback(self, char_value: str | None, **_kw: dict[str, Any]) -> None:
         """
         Called when the instrument list PV changes value.
         Args:
             char_value: The string representation of the PV data.
-            **kw: The module will also send other info about the PV, we capture this and don't use it.
+            **kw: The module will also send other info about the PV, we capture this and don't
+                use it.
         """
+        if char_value is None:
+            return
         new_inst_list = convert_inst_list(char_value)
         if new_inst_list != self.prev_inst_list:
             self.prev_inst_list = new_inst_list
             self.inst_list_changes(new_inst_list)
 
-    def remove_gatherer(self):
+    def remove_gatherer(self) -> None:
         """
         Stops the gatherer and clears the cache.
         """
@@ -88,7 +93,7 @@ def remove_gatherer(self):
             self.wait_for_gatherer_to_finish()
             self.gatherer = None
 
-    def inst_list_changes(self, inst_list):
+    def inst_list_changes(self, inst_list: InstList) -> None:
         """
         Starts a new gatherer thread.
         Args:
@@ -102,14 +107,15 @@ def inst_list_changes(self, inst_list):
         new_gatherer.start()
         self.gatherer = new_gatherer
 
-    def wait_for_gatherer_to_finish(self):
+    def wait_for_gatherer_to_finish(self) -> None:
         """
         Blocks until gatherer is finished.
         """
-        self.gatherer.join()
+        if self.gatherer is not None:
+            self.gatherer.join()
 
 
-if __name__ == "__main__":
+def main_cli() -> None:
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--cont",
@@ -135,13 +141,17 @@ def wait_for_gatherer_to_finish(self):
 
     main = InstrumentPopulatorRunner(args.cont)
     if args.as_instrument:
-        debug_inst_list = [
+        debug_inst_list: InstList = [
             {"name": args.as_instrument, "hostName": "localhost", "isScheduled": True}
         ]
         main.prev_inst_list = debug_inst_list
         main.inst_list_changes(debug_inst_list)
     elif args.test_data:
-        data = [create_web_data_with_experimenters_and_other_date([TEST_USER_1], datetime.now())]
+        data: list[RawDataEntry] = [
+            create_web_data_with_experimenters_and_other_date([TEST_USER_1], datetime.now())
+        ]
+        if not args.db_user or not args.db_pass:
+            raise ValueError("Must specify a username and password if using test data")
         update(
             "localhost",
             "localhost",
@@ -164,8 +174,15 @@ def wait_for_gatherer_to_finish(self):
                         main.remove_gatherer()
                         running = False
                     elif menu_input == "U":
-                        main.inst_list_changes(main.prev_inst_list)
+                        if main.prev_inst_list is None:
+                            logging.warning("No previous instrument list")
+                        else:
+                            main.inst_list_changes(main.prev_inst_list)
                     else:
                         logging.warning("Command not recognised: {}".format(menu_input))
         else:
             main.wait_for_gatherer_to_finish()
+
+
+if __name__ == "__main__":
+    main_cli()