Ticket8631

.github/workflows/Lint-and-test.yml

@@ -0,0 +1,42 @@
+name: Lint-and-test
+on: [pull_request, workflow_call]
+jobs:
+  call-linter-workflow:
+    uses: ISISComputingGroup/reusable-workflows/.github/workflows/linters.yml@main
+    with:
+      compare-branch: origin/master
+      python-ver: '3.11'
+  tests:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ "ubuntu-latest" ]
+        # Wide matrix of versions as this may run on a RHEL node with old python versions,
+        # but we also want it to work on dev machines
+        version: ['3.8', '3.9', '3.10', '3.11', '3.12', '3.13']
+        include:
+          - os: "windows-latest"
+            version: '3.11'
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.version }}
+      - name: install requirements
+        run: pip install -e .[dev]
+      - name: run pytest (linux)
+        run: python -m pytest
+  results:
+    if: ${{ always() }}
+    runs-on: ubuntu-latest
+    name: Final Results
+    needs: [call-linter-workflow, tests]
+    steps:
+    - run: exit 1
+      # see https://stackoverflow.com/a/67532120/4907315
+      if: >-
+          ${{
+               contains(needs.*.result, 'failure')
+            || contains(needs.*.result, 'cancelled')
+          }}

.gitignore

@@ -17,3 +17,8 @@ relPaths.sh
 *.kdb
 *.kdbx
 /exp_db_populator_venv
+/.venv
+/.coverage
+*.egg-info
+logs
+/build

create_rb_number_populator_python_venv.sh

@@ -3,6 +3,6 @@
 venv="exp_db_populator_venv" # Name of the virtual environment
 /usr/local/bin/python3.8 -m venv /home/epics/RB_num_populator/$venv # create virtual environment
 source $venv/bin/activate # activate the virtual environment
-$venv/bin/pip install -r requirements.txt # Install requirements.txt
+$venv/bin/pip install -e . # Install requirements
 deactivate # deactivate the virtual environment
 echo "Virtual environment created"

main.py → exp_db_populator/cli.py

@@ -3,6 +3,11 @@
 from datetime import datetime
 from logging.handlers import TimedRotatingFileHandler
 
+from exp_db_populator.webservices_test_data import (
+    TEST_USER_1,
+    create_web_data_with_experimenters_and_other_date,
+)
+
 # Loging must be handled here as some imports might log errors
 log_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), "logs")
 if not os.path.exists(log_folder):
@@ -23,29 +28,24 @@
 import zlib
 
 import epics
-from six.moves import input
 
 from exp_db_populator.gatherer import Gatherer
 from exp_db_populator.populator import update
 from exp_db_populator.webservices_reader import reformat_data
-from tests.webservices_test_data import (
-    TEST_USER_1,
-    create_web_data_with_experimenters_and_other_date,
-)
 
 # PV that contains the instrument list
 INST_LIST_PV = "CS:INSTLIST"
 
 
-def convert_inst_list(value_from_PV):
+def convert_inst_list(value_from_pv):
     """
     Converts the instrument list coming from the PV into a dictionary.
     Args:
-        value_from_PV: The raw value from the PV.
+        value_from_pv: The raw value from the PV.
     Returns:
         dict: The instrument information.
     """
-    json_string = zlib.decompress(bytes.fromhex(value_from_PV)).decode("utf-8")
+    json_string = zlib.decompress(bytes.fromhex(value_from_pv)).decode("utf-8")
     return json.loads(json_string)
 
 
@@ -71,7 +71,8 @@ def inst_list_callback(self, char_value, **kw):
         Called when the instrument list PV changes value.
         Args:
             char_value: The string representation of the PV data.
-            **kw: The module will also send other info about the PV, we capture this and don't use it.
+            **kw: The module will also send other info about the PV, we capture this and don't
+                use it.
         """
         new_inst_list = convert_inst_list(char_value)
         if new_inst_list != self.prev_inst_list:
@@ -109,7 +110,7 @@ def wait_for_gatherer_to_finish(self):
         self.gatherer.join()
 
 
-if __name__ == "__main__":
+def main_cli():
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--cont",
@@ -142,6 +143,8 @@ def wait_for_gatherer_to_finish(self):
         main.inst_list_changes(debug_inst_list)
     elif args.test_data:
         data = [create_web_data_with_experimenters_and_other_date([TEST_USER_1], datetime.now())]
+        if not args.db_user or not args.db_pass:
+            raise ValueError("Must specify a username and password if using test data")
         update(
             "localhost",
             "localhost",
@@ -169,3 +172,7 @@ def wait_for_gatherer_to_finish(self):
                         logging.warning("Command not recognised: {}".format(menu_input))
         else:
             main.wait_for_gatherer_to_finish()
+
+
+if __name__ == "__main__":
+    main_cli()

exp_db_populator/data_types.py

@@ -19,7 +19,8 @@ def __str__(self):
     @property
     def user_id(self):
         """
-        Gets the user id for the user. Will create an entry for the user in the database if one doesn't exist.
+        Gets the user id for the user. Will create an entry for the user in
+        the database if one doesn't exist.
 
         Returns: the user's id.
         """
@@ -43,7 +44,8 @@ def __init__(self, user, role, rb_number, start_date):
     @property
     def role_id(self):
         """
-        Gets the role id for the user based on the roles in the database. Will raise an exception if role is not found.
+        Gets the role id for the user based on the roles in the database.
+        Will raise an exception if role is not found.
 
         Returns: the role id.
 

exp_db_populator/database_model.py

@@ -1,4 +1,13 @@
-from peewee import *
+from peewee import (
+    AutoField,
+    CharField,
+    CompositeKey,
+    DateTimeField,
+    ForeignKeyField,
+    IntegerField,
+    Model,
+    Proxy,
+)
 
 # Model built using peewiz
 

exp_db_populator/gatherer.py

@@ -53,13 +53,15 @@ def run(self):
         """
         while self.running:
             all_data = gather_data()
+
             for inst in self.inst_list:
                 if inst["isScheduled"]:
                     name, host = correct_name(inst["name"]), inst["hostName"]
                     instrument_list = filter_instrument_data(all_data, name)
                     if not instrument_list:
                         logging.error(
-                            f"Unable to update {name}, no data found. Expired data will still be cleared."
+                            f"Unable to update {name}, no data found. "
+                            f"Expired data will still be cleared."
                         )
                         data_to_populate = None
                     else:

exp_db_populator/populator.py

@@ -14,8 +14,11 @@
         "unless username/password are specified manually"
     )
 
-AGE_OF_EXPIRATION = 100  # How old (in days) the startdate of an experiment must be before it is removed from the database
-POLLING_TIME = 3600  # Time in seconds between polling the website
+# How old (in days) the startdate of an experiment must be before it is removed from the database
+AGE_OF_EXPIRATION = 100
+
+# Time in seconds between polling the website
+POLLING_TIME = 3600
 
 
 def remove_users_not_referenced():
@@ -56,7 +59,8 @@ def populate(experiments, experiment_teams):
 
     Args:
         experiments (list[dict]): A list of dictionaries containing information on experiments.
-        experiment_teams (list[exp_db_populator.data_types.ExperimentTeamData]): A list containing the users for all new experiments.
+        experiment_teams (list[exp_db_populator.data_types.ExperimentTeamData]): A list containing
+            the users for all new experiments.
     """
     if not experiments or not experiment_teams:
         raise KeyError("Experiment without team or vice versa")
@@ -93,10 +97,11 @@ def update(
         instrument_name: The name of the instrument to update.
         instrument_host: The host name of the instrument to update.
         db_lock: A lock for writing to the database.
-        instrument_data: The data to send to the instrument, if None the data will just be cleared instead.
-        run_continuous: Whether or not the program is running in continuous mode.
-        credentials: The credentials to write to the database with, in the form (user, password). If None then the
-              credentials are received from the stored git repo
+        instrument_data: The data to send to the instrument, if None the data will just be
+            cleared instead.
+        run_continuous: Whether the program is running in continuous mode.
+        credentials: The credentials to write to the database with, in the form (user, password).
+            If None then the credentials are received from the stored git repo
     """
     database = create_database(instrument_host, credentials)
     logging.info(

exp_db_populator/webservices_reader.py

@@ -1,9 +1,9 @@
 # -*- coding: utf-8 -*-
 import logging
 import math
-import ssl
 from datetime import datetime, timedelta
 
+import requests
 from suds.client import Client
 
 from exp_db_populator.data_types import CREDS_GROUP, ExperimentTeamData, UserData
@@ -19,13 +19,11 @@
 RELEVANT_DATE_RANGE = 100  # How many days of data to gather (either side of now)
 DATE_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
 
-BUS_APPS_SITE = "https://api.facilities.rl.ac.uk/ws/"
-BUS_APPS_AUTH = BUS_APPS_SITE + "UserOfficeWebService?wsdl"
-BUS_APPS_API = BUS_APPS_SITE + "ScheduleWebService?wsdl"
+BUS_APPS_SITE = "https://api.facilities.rl.ac.uk/"
+BUS_APPS_AUTH = BUS_APPS_SITE + "users-service/v1/sessions"
+BUS_APPS_API = BUS_APPS_SITE + "ws/ScheduleWebService?wsdl"
 
-# This is a workaround because the web service does not have a valid certificate
-if hasattr(ssl, "_create_unverified_context"):
-    ssl._create_default_https_context = ssl._create_unverified_context
+SUCCESSFUL_LOGIN_STATUS_CODE = 201
 
 
 def get_start_and_end(date, time_range_days):
@@ -60,7 +58,15 @@ def connect():
     try:
         username, password = get_credentials(CREDS_GROUP, "WebRead")
 
-        session_id = Client(BUS_APPS_AUTH).service.login(username, password)
+        response = requests.post(BUS_APPS_AUTH, json={"username": username, "password": password})
+
+        if response.status_code != SUCCESSFUL_LOGIN_STATUS_CODE:
+            raise IOError(
+                f"Failed to authenticate to busapps web service, "
+                f"code={response.status_code}, resp={response.text}"
+            )
+
+        session_id = response.json()["sessionId"]
         client = Client(BUS_APPS_API)
 
         return client, session_id
@@ -104,8 +110,9 @@ def reformat_data(instrument_data_list):
         instrument_data_list (list): List of an instrument's data from the website.
 
     Returns:
-        tuple (list, list): A list of the experiments and their associated data and a list of the experiment teams,
-                            and a dictionary of rb_numbers and their associated instrument..
+        tuple (list, list): A list of the experiments and their associated data and a
+            list of the experiment teams, and a dictionary of rb_numbers and their associated
+            instrument.
     """
     try:
         experiments = []