Feedback from tomogram picking

pyproject.toml

@@ -107,6 +107,7 @@ GitHub = "https://github.com/DiamondLightSource/python-murfey"
 "clem.register_preprocessing_result" = "murfey.workflows.clem.register_preprocessing_results:run"
 "pato" = "murfey.workflows.notifications:notification_setup"
 "picked_particles" = "murfey.workflows.spa.picking:particles_picked"
+"picked_tomogram" = "murfey.workflows.tomo.picking:picked_tomogram"
 "spa.flush_spa_preprocess" = "murfey.workflows.spa.flush_spa_preprocess:flush_spa_preprocess"
 
 [tool.setuptools]

src/murfey/cli/inject_spa_processing.py

@@ -13,12 +13,12 @@
 from murfey.util.config import get_machine_config, get_microscope, get_security_config
 from murfey.util.db import (
     AutoProcProgram,
+    ClassificationFeedbackParameters,
     ClientEnvironment,
     DataCollection,
     DataCollectionGroup,
     Movie,
     ProcessingJob,
-    SPAFeedbackParameters,
     SPARelionParameters,
 )
 from murfey.util.processing_params import default_spa_parameters
@@ -137,9 +137,9 @@ def run():
             .where(ProcessingJob.recipe == "em-spa-preprocess")
         ).one()
         params = murfey_db.exec(
-            select(SPARelionParameters, SPAFeedbackParameters)
+            select(SPARelionParameters, ClassificationFeedbackParameters)
             .where(SPARelionParameters.pj_id == collected_ids[2].id)
-            .where(SPAFeedbackParameters.pj_id == SPARelionParameters.pj_id)
+            .where(ClassificationFeedbackParameters.pj_id == SPARelionParameters.pj_id)
         ).one()
         proc_params: dict | None = dict(params[0])
         feedback_params = params[1]

src/murfey/cli/spa_ispyb_messages.py

@@ -363,7 +363,7 @@ def run():
         small_boxsize=metadata["small_boxsize"],
         mask_diameter=metadata["mask_diameter"],
     )
-    feedback_params = db.SPAFeedbackParameters(
+    feedback_params = db.ClassificationFeedbackParameters(
         pj_id=collected_ids[2].id,
         estimate_particle_diameter=not bool(metadata["particle_diameter"]),
         hold_class2d=False,

src/murfey/client/contexts/tomo.py

@@ -161,7 +161,13 @@ def register_tomography_data_collections(
                             data=dc_data,
                         )
 
-                        for recipe in ("em-tomo-preprocess", "em-tomo-align"):
+                        recipes_to_assign_pjids = [
+                            "em-tomo-preprocess",
+                            "em-tomo-align",
+                        ]
+                        if not self._tilt_series_with_pjids:
+                            recipes_to_assign_pjids.append("em-tomo-class2d")
+                        for recipe in recipes_to_assign_pjids:
                             capture_post(
                                 base_url=str(environment.url.geturl()),
                                 router_name="workflow.router",

src/murfey/server/api/session_info.py

@@ -31,6 +31,7 @@
 from murfey.util import sanitise
 from murfey.util.config import MachineConfig, get_machine_config
 from murfey.util.db import (
+    ClassificationFeedbackParameters,
     ClientEnvironment,
     DataCollection,
     DataCollectionGroup,
@@ -41,7 +42,6 @@
     RsyncInstance,
     Session,
     SessionProcessingParameters,
-    SPAFeedbackParameters,
     SPARelionParameters,
     Tilt,
     TiltSeries,
@@ -280,7 +280,7 @@ class ProcessingDetails(BaseModel):
     data_collections: List[DataCollection]
     processing_jobs: List[ProcessingJob]
     relion_params: SPARelionParameters
-    feedback_params: SPAFeedbackParameters
+    feedback_params: ClassificationFeedbackParameters
 
 
 @spa_router.get("/sessions/{session_id}/spa_processing_parameters")
@@ -293,13 +293,13 @@ def get_spa_proc_param_details(
             DataCollection,
             ProcessingJob,
             SPARelionParameters,
-            SPAFeedbackParameters,
+            ClassificationFeedbackParameters,
         )
         .where(DataCollectionGroup.session_id == session_id)
         .where(DataCollectionGroup.id == DataCollection.dcg_id)
         .where(DataCollection.id == ProcessingJob.dc_id)
         .where(SPARelionParameters.pj_id == ProcessingJob.id)
-        .where(SPAFeedbackParameters.pj_id == ProcessingJob.id)
+        .where(ClassificationFeedbackParameters.pj_id == ProcessingJob.id)
     ).all()
     if not params:
         return None

src/murfey/server/api/workflow.py

@@ -44,6 +44,7 @@
 from murfey.util.config import get_machine_config
 from murfey.util.db import (
     AutoProcProgram,
+    ClassificationFeedbackParameters,
     DataCollection,
     DataCollectionGroup,
     FoilHole,
@@ -54,7 +55,6 @@
     SearchMap,
     Session,
     SessionProcessingParameters,
-    SPAFeedbackParameters,
     SPARelionParameters,
     Tilt,
     TiltSeries,
@@ -409,9 +409,9 @@ async def request_spa_preprocessing(
             .where(ProcessingJob.recipe == "em-spa-preprocess")
         ).one()
         params = db.exec(
-            select(SPARelionParameters, SPAFeedbackParameters)
+            select(SPARelionParameters, ClassificationFeedbackParameters)
             .where(SPARelionParameters.pj_id == collected_ids[2].id)
-            .where(SPAFeedbackParameters.pj_id == SPARelionParameters.pj_id)
+            .where(ClassificationFeedbackParameters.pj_id == SPARelionParameters.pj_id)
         ).one()
         proc_params: Optional[dict] = dict(params[0])
         feedback_params = params[1]

src/murfey/server/demo_api.py

@@ -49,6 +49,7 @@
 )
 from murfey.util.db import (
     AutoProcProgram,
+    ClassificationFeedbackParameters,
     ClientEnvironment,
     DataCollection,
     DataCollectionGroup,
@@ -60,7 +61,6 @@
     ProcessingJob,
     RsyncInstance,
     Session,
-    SPAFeedbackParameters,
     SPARelionParameters,
     Tilt,
     TiltSeries,
@@ -244,7 +244,7 @@ class ProcessingDetails(BaseModel):
     data_collections: List[DataCollection]
     processing_jobs: List[ProcessingJob]
     relion_params: SPARelionParameters
-    feedback_params: SPAFeedbackParameters
+    feedback_params: ClassificationFeedbackParameters
 
 
 @router.get("/sessions/{session_id}/spa_processing_parameters")
@@ -257,13 +257,13 @@ def get_spa_proc_param_details(
             DataCollection,
             ProcessingJob,
             SPARelionParameters,
-            SPAFeedbackParameters,
+            ClassificationFeedbackParameters,
         )
         .where(DataCollectionGroup.session_id == session_id)
         .where(DataCollectionGroup.id == DataCollection.dcg_id)
         .where(DataCollection.id == ProcessingJob.dc_id)
         .where(SPARelionParameters.pj_id == ProcessingJob.id)
-        .where(SPAFeedbackParameters.pj_id == ProcessingJob.id)
+        .where(ClassificationFeedbackParameters.pj_id == ProcessingJob.id)
     ).all()
     if not params:
         return None
@@ -560,9 +560,9 @@ def flush_spa_processing(
         .where(ProcessingJob.recipe == "em-spa-preprocess")
     ).one()
     params = db.exec(
-        select(SPARelionParameters, SPAFeedbackParameters)
+        select(SPARelionParameters, ClassificationFeedbackParameters)
         .where(SPARelionParameters.pj_id == collected_ids[2].id)
-        .where(SPAFeedbackParameters.pj_id == SPARelionParameters.pj_id)
+        .where(ClassificationFeedbackParameters.pj_id == SPARelionParameters.pj_id)
     ).one()
     proc_params = dict(params[0])
     feedback_params = params[1]

src/murfey/server/feedback.py

@@ -313,13 +313,15 @@ def _pj_id(app_id: int, _db, recipe: str = "") -> int:
 
 def _get_spa_params(
     app_id: int, _db
-) -> Tuple[db.SPARelionParameters, db.SPAFeedbackParameters]:
+) -> Tuple[db.SPARelionParameters, db.ClassificationFeedbackParameters]:
     pj_id = _pj_id(app_id, _db, recipe="em-spa-preprocess")
     relion_params = _db.exec(
         select(db.SPARelionParameters).where(db.SPARelionParameters.pj_id == pj_id)
     ).one()
     feedback_params = _db.exec(
-        select(db.SPAFeedbackParameters).where(db.SPAFeedbackParameters.pj_id == pj_id)
+        select(db.ClassificationFeedbackParameters).where(
+            db.ClassificationFeedbackParameters.pj_id == pj_id
+        )
     ).one()
     _db.expunge(relion_params)
     _db.expunge(feedback_params)
@@ -412,8 +414,8 @@ def _release_3d_hold(message: dict, _db):
         )
     ).one()
     feedback_params = _db.exec(
-        select(db.SPAFeedbackParameters).where(
-            db.SPAFeedbackParameters.pj_id == pj_id_params
+        select(db.ClassificationFeedbackParameters).where(
+            db.ClassificationFeedbackParameters.pj_id == pj_id_params
         )
     ).one()
     class3d_params = _db.exec(
@@ -490,8 +492,8 @@ def _release_refine_hold(message: dict, _db):
         )
     ).one()
     feedback_params = _db.exec(
-        select(db.SPAFeedbackParameters).where(
-            db.SPAFeedbackParameters.pj_id == pj_id_params
+        select(db.ClassificationFeedbackParameters).where(
+            db.ClassificationFeedbackParameters.pj_id == pj_id_params
         )
     ).one()
     refine_params = _db.exec(
@@ -582,8 +584,8 @@ def _register_incomplete_2d_batch(message: dict, _db, demo: bool = False):
         )
     ).one()
     feedback_params = _db.exec(
-        select(db.SPAFeedbackParameters).where(
-            db.SPAFeedbackParameters.pj_id == pj_id_params
+        select(db.ClassificationFeedbackParameters).where(
+            db.ClassificationFeedbackParameters.pj_id == pj_id_params
         )
     ).one()
     if feedback_params.hold_class2d:
@@ -623,7 +625,9 @@ def _register_incomplete_2d_batch(message: dict, _db, demo: bool = False):
         )
         _db.add(class2d_params)
         _db.commit()
-        murfey_ids = _murfey_id(message["program_id"], _db, number=50)
+        murfey_ids = _murfey_id(
+            message["program_id"], _db, number=default_spa_parameters.nr_classes_2d
+        )
         _murfey_class2ds(
             murfey_ids, class2d_message["particles_file"], message["program_id"], _db
         )
@@ -706,8 +710,8 @@ def _register_complete_2d_batch(message: dict, _db, demo: bool = False):
         )
     ).one()
     feedback_params = _db.exec(
-        select(db.SPAFeedbackParameters).where(
-            db.SPAFeedbackParameters.pj_id == pj_id_params
+        select(db.ClassificationFeedbackParameters).where(
+            db.ClassificationFeedbackParameters.pj_id == pj_id_params
         )
     ).one()
     _db.expunge(relion_params)
@@ -747,7 +751,9 @@ def _register_complete_2d_batch(message: dict, _db, demo: bool = False):
             _db.add(class2d_params)
             _db.commit()
             _db.close()
-            murfey_ids = _murfey_id(_app_id(pj_id, _db), _db, number=50)
+            murfey_ids = _murfey_id(
+                _app_id(pj_id, _db), _db, number=default_spa_parameters.nr_classes_2d
+            )
             _murfey_class2ds(
                 murfey_ids, class2d_message["particles_file"], _app_id(pj_id, _db), _db
             )
@@ -796,7 +802,13 @@ def _register_complete_2d_batch(message: dict, _db, demo: bool = False):
         else:
             class_uuids = {
                 str(i + 1): m
-                for i, m in enumerate(_murfey_id(_app_id(pj_id, _db), _db, number=50))
+                for i, m in enumerate(
+                    _murfey_id(
+                        _app_id(pj_id, _db),
+                        _db,
+                        number=default_spa_parameters.nr_classes_2d,
+                    )
+                )
             }
             class2d_grp_uuid = _murfey_id(_app_id(pj_id, _db), _db)[0]
         zocalo_message: dict = {
@@ -865,7 +877,13 @@ def _register_complete_2d_batch(message: dict, _db, demo: bool = False):
         else:
             class_uuids = {
                 str(i + 1): m
-                for i, m in enumerate(_murfey_id(_app_id(pj_id, _db), _db, number=50))
+                for i, m in enumerate(
+                    _murfey_id(
+                        _app_id(pj_id, _db),
+                        _db,
+                        number=default_spa_parameters.nr_classes_2d,
+                    )
+                )
             }
             class2d_grp_uuid = _murfey_id(_app_id(pj_id, _db), _db)[0]
         zocalo_message = {
@@ -913,7 +931,7 @@ def _flush_class2d(
     app_id: int,
     _db,
     relion_params: db.SPARelionParameters | None = None,
-    feedback_params: db.SPAFeedbackParameters | None = None,
+    feedback_params: db.ClassificationFeedbackParameters | None = None,
 ):
     instrument_name = (
         _db.exec(select(db.Session).where(db.Session.id == session_id))
@@ -934,8 +952,8 @@ def _flush_class2d(
         _db.expunge(relion_params)
     if not feedback_params:
         feedback_params = _db.exec(
-            select(db.SPAFeedbackParameters).where(
-                db.SPAFeedbackParameters.pj_id == pj_id_params
+            select(db.ClassificationFeedbackParameters).where(
+                db.ClassificationFeedbackParameters.pj_id == pj_id_params
             )
         ).one()
         _db.expunge(feedback_params)
@@ -1012,8 +1030,8 @@ def _register_class_selection(message: dict, _db, demo: bool = False):
     ).all()
     # Add the class selection score to the database
     feedback_params = _db.exec(
-        select(db.SPAFeedbackParameters).where(
-            db.SPAFeedbackParameters.pj_id == pj_id_params
+        select(db.ClassificationFeedbackParameters).where(
+            db.ClassificationFeedbackParameters.pj_id == pj_id_params
         )
     ).one()
     _db.expunge(feedback_params)
@@ -1233,8 +1251,8 @@ def _register_3d_batch(message: dict, _db, demo: bool = False):
     ).one()
     relion_options = dict(relion_params)
     feedback_params = _db.exec(
-        select(db.SPAFeedbackParameters).where(
-            db.SPAFeedbackParameters.pj_id == pj_id_params
+        select(db.ClassificationFeedbackParameters).where(
+            db.ClassificationFeedbackParameters.pj_id == pj_id_params
         )
     ).one()
     other_options = dict(feedback_params)
@@ -1411,8 +1429,8 @@ def _register_initial_model(message: dict, _db, demo: bool = False):
     pj_id_params = _pj_id(message["program_id"], _db, recipe="em-spa-preprocess")
     # Add the initial model file to the database
     feedback_params = _db.exec(
-        select(db.SPAFeedbackParameters).where(
-            db.SPAFeedbackParameters.pj_id == pj_id_params
+        select(db.ClassificationFeedbackParameters).where(
+            db.ClassificationFeedbackParameters.pj_id == pj_id_params
         )
     ).one()
     feedback_params.initial_model = message.get("initial_model")
@@ -1578,8 +1596,8 @@ def _register_refinement(message: dict, _db, demo: bool = False):
     ).one()
     relion_options = dict(relion_params)
     feedback_params = _db.exec(
-        select(db.SPAFeedbackParameters).where(
-            db.SPAFeedbackParameters.pj_id == pj_id_params
+        select(db.ClassificationFeedbackParameters).where(
+            db.ClassificationFeedbackParameters.pj_id == pj_id_params
         )
     ).one()
     other_options = dict(feedback_params)
@@ -1726,8 +1744,8 @@ def _register_bfactors(message: dict, _db, demo: bool = False):
     ).one()
     relion_options = dict(relion_params)
     feedback_params = _db.exec(
-        select(db.SPAFeedbackParameters).where(
-            db.SPAFeedbackParameters.pj_id == pj_id_params
+        select(db.ClassificationFeedbackParameters).where(
+            db.ClassificationFeedbackParameters.pj_id == pj_id_params
         )
     ).one()
 
@@ -2289,7 +2307,7 @@ def feedback_callback(header: dict, message: dict, _db=murfey_db) -> None:
                     eer_fractionation_file=message["eer_fractionation_file"],
                     symmetry=message["symmetry"],
                 )
-                feedback_params = db.SPAFeedbackParameters(
+                feedback_params = db.ClassificationFeedbackParameters(
                     pj_id=collected_ids[2].id,
                     estimate_particle_diameter=True,
                     hold_class2d=False,
@@ -2346,7 +2364,18 @@ def feedback_callback(header: dict, message: dict, _db=murfey_db) -> None:
                     gain_ref=message["gain_ref"],
                     eer_fractionation_file=message["eer_fractionation_file"],
                 )
+                feedback_params = db.ClassificationFeedbackParameters(
+                    pj_id=collected_ids[2].id,
+                    estimate_particle_diameter=True,
+                    hold_class2d=False,
+                    hold_class3d=False,
+                    class_selection_score=0,
+                    star_combination_job=0,
+                    initial_model="",
+                    next_job=0,
+                )
                 _db.add(params)
+                _db.add(feedback_params)
                 _db.commit()
                 _db.close()
             if murfey.server._transport_object: