diff --git a/specifyweb/backend/interactions/tests/test_modify_update_of_interaction_sibling_preps.py b/specifyweb/backend/interactions/tests/test_modify_update_of_interaction_sibling_preps.py index f5cf417c3f3..54ce324aab7 100644 --- a/specifyweb/backend/interactions/tests/test_modify_update_of_interaction_sibling_preps.py +++ b/specifyweb/backend/interactions/tests/test_modify_update_of_interaction_sibling_preps.py @@ -4,11 +4,11 @@ from specifyweb.backend.interactions.tests.test_cog_consolidated_prep_sibling_context import ( TestCogConsolidatedPrepSiblingContext, ) -from specifyweb.specify.api.api_utils import strict_uri_to_model +from specifyweb.specify.api.serializers import obj_to_data +from specifyweb.specify.api_utils import strict_uri_to_model from specifyweb.specify.models import ( Borrow, Disposal, - Disposalpreparation, Gift, Giftpreparation, Loan, @@ -16,8 +16,6 @@ ) import copy -from specifyweb.specify.api.serializers import obj_to_data - PrepGetter = Callable[["TestModifyUpdateInteractionSiblingPreps"], list[Any]] PrepGetterFromPreps = Callable[[list[Any]], list[Any]] diff --git a/specifyweb/backend/interactions/views.py b/specifyweb/backend/interactions/views.py index 20434b10785..59b63c4764b 100644 --- a/specifyweb/backend/interactions/views.py +++ b/specifyweb/backend/interactions/views.py @@ -15,13 +15,11 @@ from specifyweb.backend.permissions.permissions import check_table_permissions, table_permissions_checker from specifyweb.specify.api.api_utils import strict_uri_to_model from specifyweb.specify.models import Collectionobject, Loan, Loanpreparation, \ - Loanreturnpreparation, Preparation, Recordset, Recordsetitem + Loanreturnpreparation, Preparation, Recordset from specifyweb.specify.api.serializers import toJson from specifyweb.specify.views import login_maybe_required -from django.db.models import F, Q, Sum -from django.db.models.functions import Coalesce -from django.http import JsonResponse +from django.db.models import Q @require_POST # NOTE: why is this a POST request? @login_maybe_required diff --git a/specifyweb/backend/setup_tool/api.py b/specifyweb/backend/setup_tool/api.py new file mode 100644 index 00000000000..15f2f35e6e8 --- /dev/null +++ b/specifyweb/backend/setup_tool/api.py @@ -0,0 +1,389 @@ +import json +from django.http import (JsonResponse) +from django.db.models import Max +from django.db import transaction + +from specifyweb.backend.permissions.models import UserPolicy +from specifyweb.specify.models import Spversion +from specifyweb.specify import models +from specifyweb.backend.setup_tool.utils import normalize_keys, resolve_uri_or_fallback +from specifyweb.backend.setup_tool.schema_defaults import apply_schema_defaults +from specifyweb.backend.setup_tool.picklist_defaults import create_default_picklists +from specifyweb.backend.setup_tool.prep_type_defaults import create_default_prep_types +from specifyweb.backend.setup_tool.setup_tasks import setup_database_background, get_active_setup_task, get_last_setup_error, set_last_setup_error, MissingWorkerError +from specifyweb.backend.setup_tool.tree_defaults import create_default_tree, update_tree_scoping +from specifyweb.specify.models import Institution, Discipline + +import logging +logger = logging.getLogger(__name__) + +APP_VERSION = "7" +SCHEMA_VERSION = "2.10" + +class SetupError(Exception): + """Raised by any setup tasks.""" + pass + +def get_setup_progress() -> dict: + """Returns a dictionary of the status of the database setup.""" + # Check if setup is currently in progress + active_setup_task, busy = get_active_setup_task() + + completed_resources = None + last_error = None + # Get setup progress if its currently in progress. + if active_setup_task: + info = getattr(active_setup_task, "info", None) or getattr(active_setup_task, "result", None) + if isinstance(info, dict): + completed_resources = info.get('progress', None) + last_error = info.get('error', get_last_setup_error()) + if last_error is not None: + set_last_setup_error(last_error) + + if completed_resources is None: + completed_resources = get_setup_resource_progress() + last_error = get_last_setup_error() + + return { + "resources": completed_resources, + "last_error": last_error, + "busy": busy, + } + +def get_setup_resource_progress() -> dict: + """Returns a dictionary of the status of database setup resources.""" + institution_created = models.Institution.objects.exists() + institution = models.Institution.objects.first() + globalGeographyTree = institution and institution.issinglegeographytree + + return { + "institution": institution_created, + "storageTreeDef": models.Storagetreedef.objects.exists(), + "globalGeographyTreeDef": institution_created and ((not globalGeographyTree) or models.Geographytreedef.objects.exists()), + "division": models.Division.objects.exists(), + "discipline": models.Discipline.objects.exists(), + "geographyTreeDef": models.Geographytreedef.objects.exists(), + "taxonTreeDef": models.Taxontreedef.objects.exists(), + "collection": models.Collection.objects.exists(), + "specifyUser": models.Specifyuser.objects.exists(), + } + +def _guided_setup_condition(request) -> bool: + from specifyweb.specify.models import Specifyuser + if Specifyuser.objects.exists(): + is_auth = request.user.is_authenticated + user = Specifyuser.objects.filter(id=request.user.id).first() + if not user or not is_auth or not user.usertype in ('Admin', 'Manager'): + return False + return True + +def handle_request(request, create_resource, direct=False): + """Generic handler for any setup resource POST request.""" + # Check permission and only allow POST requests + if not _guided_setup_condition(request): + return JsonResponse({"error": "Not permitted"}, status=401) + if request.method != 'POST': + return JsonResponse({"error": "Invalid request"}, status=400) + + raw_data = json.loads(request.body) + data = normalize_keys(raw_data) + + try: + response = create_resource(data) + return JsonResponse({"success": True, "setup_progress": get_setup_progress(), **response}, status=200) + + except Exception as e: + return JsonResponse({"error": str(e)}, status=400) + +def setup_database(request, direct=False): + """Creates all database setup resources sequentially in the background. Atomic.""" + # Check permission and only allow POST requests + if not _guided_setup_condition(request): + return JsonResponse({"error": "Not permitted"}, status=401) + if request.method != 'POST': + return JsonResponse({"error": "Invalid request"}, status=400) + + # Check that there isn't another setup task running. + active_setup_task, busy = get_active_setup_task() + if busy: + return JsonResponse({"error": "Database setup is already in progress."}, status=400) + + try: + logger.debug("Starting Database Setup.") + raw_data = json.loads(request.body) + data = normalize_keys(raw_data) + + task_id = setup_database_background(data) + + logger.debug("Database setup started successfully.") + return JsonResponse({"success": True, "setup_progress": get_setup_progress(), "task_id": task_id}, status=200) + except MissingWorkerError as e: + logger.exception(str(e)) + return JsonResponse({'error': str(e)}, status=500) + except Exception as e: + logger.exception(str(e)) + return JsonResponse({'error': 'An internal server error occurred.'}, status=500) + +def create_institution(data): + from specifyweb.specify.models import Institution, Address + + # Check that there are no institutions. Only one should ever exist. + if Institution.objects.count() > 0: + raise SetupError('An institution already exists, cannot create another.') + + # Get address fields (if any) + address_data = data.pop('address', None) + + # New DB: force id = 1 + data['id'] = 1 + + # Create address + with transaction.atomic(): + if address_data: + address_obj = Address.objects.create(**address_data) + data['address_id'] = address_obj.id + + # Create institution + new_institution = Institution.objects.create(**data) + Spversion.objects.create(appversion=APP_VERSION, schemaversion=SCHEMA_VERSION) + return {'institution_id': new_institution} + +def create_division(data): + from specifyweb.specify.models import Division, Institution + + # If division_id is provided and exists, return success + existing_id = data.pop('division_id', None) + if existing_id: + existing_division = Division.objects.filter(id=existing_id).first() + if existing_division: + return {"division_id": existing_division.id} + + # Determine new Division ID + max_id = Division.objects.aggregate(Max('id'))['id__max'] or 0 + data['id'] = max_id + 1 + + # Normalize abbreviation + data['abbrev'] = data.pop('abbreviation', None) or data.get('abbrev', '') + + # Handle institution assignment + institution_url = data.pop('institution', None) + institution = resolve_uri_or_fallback(institution_url, None, Institution) + data['institution_id'] = institution.id if institution else None + + # Remove unwanted keys + for key in ['_tableName', 'success']: + data.pop(key, None) + + # Create new division + try: + new_division = Division.objects.create(**data) + return {"division_id": new_division.id} + except Exception as e: + logger.exception(f'Division error: {e}') + raise SetupError(e) + +def create_discipline(data): + from specifyweb.specify.models import ( + Division, Datatype, Geographytreedef, + Geologictimeperiodtreedef + ) + + # Check if discipline_id is provided and already exists + existing_id = data.pop('discipline_id', None) + if existing_id: + existing_discipline = Discipline.objects.filter(id=existing_id).first() + if existing_discipline: + return {"discipline_id": existing_discipline.id} + + # Resolve division + division_url = data.get('division') + division = resolve_uri_or_fallback(division_url, None, Division) + if not division: + raise SetupError("No Division available to assign") + + data['division'] = division + + # Ensure required foreign key objects exist + datatype = Datatype.objects.last() or Datatype.objects.create(id=1, name='Biota') + geographytreedef_url = data.pop('geographytreedef', None) + geologictimeperiodtreedef_url = data.pop('geologictimeperiodtreedef', None) + geographytreedef = resolve_uri_or_fallback(geographytreedef_url, None, Geographytreedef) + geologictimeperiodtreedef = resolve_uri_or_fallback(geologictimeperiodtreedef_url, None, Geologictimeperiodtreedef) + + if geographytreedef is None or geologictimeperiodtreedef is None: + raise SetupError("A Geography tree and Chronostratigraphy tree must exist before creating an institution.") + + data.update({ + 'datatype_id': datatype.id, + 'geographytreedef_id': geographytreedef.id, + 'geologictimeperiodtreedef_id': geologictimeperiodtreedef.id + }) + + # Assign new Discipline ID + max_id = Discipline.objects.aggregate(Max('id'))['id__max'] or 0 + data['id'] = max_id + 1 + + # Remove unwanted keys + for key in ['_tablename', 'success', 'datatype']: + data.pop(key, None) + + # Create new Discipline + try: + new_discipline = Discipline.objects.create(**data) + + # Check if initial setup. + if not division_url: + # Create Splocalecontainers for all datamodel tables + apply_schema_defaults(new_discipline) + + # Update tree scoping + update_tree_scoping(geographytreedef, new_discipline.id) + update_tree_scoping(geologictimeperiodtreedef, new_discipline.id) + + return {"discipline_id": new_discipline.id} + + except Exception as e: + raise SetupError(e) + +def create_collection(data): + from specifyweb.specify.models import Collection, Discipline + + # If collection_id is provided and exists, return success + existing_id = data.pop('collection_id', None) + if existing_id: + existing_collection = Collection.objects.filter(id=existing_id).first() + if existing_collection: + return {"collection_id": existing_collection.id} + + # Assign new Collection ID + max_id = Collection.objects.aggregate(Max('id'))['id__max'] or 0 + data['id'] = max_id + 1 + + # Handle discipline reference from URL + discipline_id = data.get('discipline_id', None) + discipline_url = data.pop('discipline', None) + discipline = resolve_uri_or_fallback(discipline_url, discipline_id, Discipline) + if discipline is not None: + data['discipline_id'] = discipline.id + else: + raise SetupError(f"No discipline available") + + # Remove keys that should not be passed to model + for key in ['_tablename', 'success']: + data.pop(key, None) + + # Create new Collection + try: + new_collection = Collection.objects.create(**data) + + # Create Preparation Types + create_default_prep_types(new_collection, discipline.type) + # Create picklists + create_default_picklists(new_collection, discipline.type) + # Create Collection Object Type + # TODO + + return {"collection_id": new_collection.id} + except Exception as e: + raise SetupError(e) + +def create_specifyuser(data): + from specifyweb.specify.models import Specifyuser, Agent, Division, Collection + + # Assign ID manually + max_id = Specifyuser.objects.aggregate(Max('id'))['id__max'] or 0 + data['id'] = max_id + 1 + + # Ensure there is an Agent + agent = Agent.objects.last() + if not agent: + agent = Agent.objects.create( + id=1, + agenttype=1, + firstname='spadmin', + division=Division.objects.last(), + ) + + try: + # Create user + new_user = Specifyuser.objects.create(**data) + new_user.set_password(new_user.password) + new_user.save() + + # Grant permissions + UserPolicy.objects.create( + specifyuser=new_user, + collection=Collection.objects.last(), + resource='%', + action='%' + ) + + # Link agent to user + agent.specifyuser = new_user + agent.save() + + return {"user_id": new_user.id} + + except Exception as e: + raise SetupError(e) + +# Trees +def create_storage_tree(data): + return create_tree('Storage', data) + +def create_global_geography_tree(data): + return create_tree('Geography', data) + +def create_geography_tree(data): + return create_tree('Geography', data) + +def create_taxon_tree(data): + return create_tree('Taxon', data) + +def create_geologictimeperiod_tree(data): + return create_tree('Geologictimeperiod', data) + +def create_lithostrat_tree(data): + return create_tree('Lithostrat', data) + +def create_tectonicunit_tree(data): + return create_tree('Tectonicunit', data) + +def create_tree(name: str, data: dict) -> dict: + # TODO: Use trees/create_default_trees + # https://github.com/specify/specify7/pull/6429 + + # Figure out which scoping field should be used. + use_institution = False + use_discipline = True + if name == 'Storage': + use_institution = True + use_discipline = False + + # Handle institution assignment + institution = None + if use_institution: + institution_url = data.pop('institution', None) + institution = resolve_uri_or_fallback(institution_url, None, Institution) + + # Handle discipline reference from URL + discipline = None + if use_discipline: + discipline_url = data.get('discipline', None) + discipline = resolve_uri_or_fallback(discipline_url, None, Discipline) + + # Get tree configuration + ranks = data.pop('ranks', dict()) + + try: + kwargs = {} + kwargs['fullnamedirection'] = data.get('fullnamedirection', 1) + if use_institution: + kwargs['institution'] = institution + if use_discipline and discipline is not None: + kwargs['discipline'] = discipline + + treedef = create_default_tree(name, kwargs, ranks) + return {'treedef_id': treedef.id} + except Exception as e: + raise SetupError(e) \ No newline at end of file diff --git a/specifyweb/backend/setup_tool/picklist_defaults.py b/specifyweb/backend/setup_tool/picklist_defaults.py new file mode 100644 index 00000000000..dab61bb9e05 --- /dev/null +++ b/specifyweb/backend/setup_tool/picklist_defaults.py @@ -0,0 +1,86 @@ +import json +from pathlib import Path +from django.db import transaction +from .utils import load_json_from_file + +import logging + +from specifyweb.specify.models import ( + Collection, + Picklist, + Picklistitem +) + +logger = logging.getLogger(__name__) + +def create_default_picklists(collection: Collection, discipline_type: str | None): + """ + Creates defaults picklists for -one- collection, including discipline specific picklists. + """ + # Global picklists + logger.debug('Creating default global picklists.') + # Get from defaults file + defaults = load_json_from_file(Path(__file__).parent.parent.parent.parent / 'config' / 'common' / 'global_picklists.json') + + global_picklists = defaults.get('picklists', None) if defaults is not None else None + if global_picklists is None: + logger.exception('No global picklists found in global_picklists.json.') + return + create_picklists(global_picklists, collection) + + # Create discipline picklists + logger.debug('Creating default discipline picklists.') + if discipline_type is None: + return + discipline_defaults = load_json_from_file(Path(__file__).parent.parent.parent.parent / 'config' / 'common' / 'picklists.json') + + discipline_picklists = discipline_defaults.get(discipline_type, None) + if discipline_picklists is None: + logger.exception(f'No picklists found for discipline "{discipline_type}" in picklists.json.') + return + create_picklists(discipline_picklists, collection) + +def create_picklists(configuraton: list, collection: Collection): + """ + Create a set of picklists from a configuration list. + """ + # Create picklists from a list of picklist configuration dicts. + try: + with transaction.atomic(): + # Create picklists in bulk + picklists_bulk = [] + for picklist in configuraton: + picklists_bulk.append( + Picklist( + collection=collection, + name=picklist.get('name'), + issystem=picklist.get('issystem', True), + type=picklist.get('type'), + tablename=picklist.get('tablename'), + fieldname=picklist.get('fieldname'), + readonly=picklist.get('readonly'), + sizelimit=picklist.get('sizelimit'), + ) + ) + Picklist.objects.bulk_create(picklists_bulk, ignore_conflicts=True) + + # Create picklist items in bulk + names = [p.name for p in picklists_bulk] + picklist_records = Picklist.objects.filter(name__in=names) + name_to_obj = {pl.name: pl for pl in picklist_records} + + picklistitems_bulk = [] + for picklist in configuraton: + parent = name_to_obj.get(picklist['name']) + for picklistitem in picklist['items']: + picklistitems_bulk.append( + Picklistitem( + picklist=parent, + title=picklistitem.get('title'), + value=picklistitem.get('value'), + ) + ) + Picklistitem.objects.bulk_create(picklistitems_bulk, ignore_conflicts=True) + + except Exception: + logger.exception('An error occured when creating default picklists.') \ No newline at end of file diff --git a/specifyweb/backend/setup_tool/prep_type_defaults.py b/specifyweb/backend/setup_tool/prep_type_defaults.py new file mode 100644 index 00000000000..0b6b8058f70 --- /dev/null +++ b/specifyweb/backend/setup_tool/prep_type_defaults.py @@ -0,0 +1,47 @@ +import json +from pathlib import Path + +import logging + +from specifyweb.specify.models import ( + Collection, + Preptype +) + +logger = logging.getLogger(__name__) + +def create_default_prep_types(collection: Collection, discipline_type: str): + """ + Load default collection prep types from the prep_types file. + """ + logger.debug('Creating default prep types.') + prep_type_list = None + prep_types_file = (Path(__file__).parent.parent.parent.parent / 'config' / 'common' / 'prep_types.json') + try: + with prep_types_file.open('r', encoding='utf-8') as fh: + prep_type_list = json.load(fh) + except Exception as e: + logger.exception(f'Failed to prepTypes from {prep_types_file}: {e}') + prep_type_list = None + + if prep_type_list is None: + return + + # Get prep types for this collection's discipline type. + prep_type_discipline_list = prep_type_list.get(discipline_type, None) + + if prep_type_discipline_list is None: + return + + prep_type_bulk = [] + for prep_type in prep_type_discipline_list: + prep_type_bulk.append( + Preptype( + collection=collection, + name=prep_type.get('name'), + isloanable=prep_type.get('isloanable') + ) + ) + + Preptype.objects.bulk_create(prep_type_bulk, ignore_conflicts=True) + \ No newline at end of file diff --git a/specifyweb/backend/setup_tool/schema_defaults.py b/specifyweb/backend/setup_tool/schema_defaults.py new file mode 100644 index 00000000000..9e93b6c651f --- /dev/null +++ b/specifyweb/backend/setup_tool/schema_defaults.py @@ -0,0 +1,44 @@ +from typing import Optional +from specifyweb.specify.models_utils.models_by_table_id import model_names_by_table_id +from specifyweb.specify.migration_utils.update_schema_config import update_table_schema_config_with_defaults +from .utils import load_json_from_file +from specifyweb.specify.models import Discipline + +from pathlib import Path + +import logging +logger = logging.getLogger(__name__) + +def apply_schema_defaults(discipline: Discipline): + """ + Apply schema config localization defaults for this discipline. + """ + # Get default schema localization + defaults = load_json_from_file(Path(__file__).parent.parent.parent.parent / 'config' / 'common' / 'schema_localization_en.json') + + # Read schema overrides file for the discipline, if it exists + schema_overrides_path = Path(__file__).parent.parent.parent.parent / 'config' / discipline.type / 'schema_overrides.json' + overrides = None + if schema_overrides_path.exists(): + load_json_from_file(schema_overrides_path) + + # Update the schema for each table individually. + for model_name in model_names_by_table_id.values(): + logger.debug(f'Applying schema defaults for {model_name}. Using overrides: {overrides is not None}.') + + # Table information + table_description = get_table_override(defaults, model_name, 'desc') + + update_table_schema_config_with_defaults( + table_name=model_name, + description=table_description, + discipline_id=discipline.id, + defaults=defaults, + overrides=overrides, + ) + +def get_table_override(overrides: Optional[dict], model_name: str, key: str): + """Get a specific table's field override from a dict of all table overrides.""" + if overrides is not None and overrides.get(model_name, None) is not None: + return overrides[model_name].get(key, None) + return None \ No newline at end of file diff --git a/specifyweb/backend/setup_tool/setup_tasks.py b/specifyweb/backend/setup_tool/setup_tasks.py new file mode 100644 index 00000000000..f52a04f5841 --- /dev/null +++ b/specifyweb/backend/setup_tool/setup_tasks.py @@ -0,0 +1,152 @@ +from specifyweb.celery_tasks import LogErrorsTask, app +from typing import Tuple, Optional +from celery.result import AsyncResult +from specifyweb.backend.setup_tool import api +from django.db import transaction +import threading +from specifyweb.specify.models_utils.model_extras import PALEO_DISCIPLINES, GEOLOGY_DISCIPLINES + +from uuid import uuid4 +import logging +logger = logging.getLogger(__name__) + +# Keep track of the currently running setup task. There should only ever be one. +_active_setup_task_id: Optional[str] = None +_active_setup_lock = threading.Lock() + +# Keep track of last error. +_last_error: Optional[str] = None +_last_error_lock = threading.Lock() + +class MissingWorkerError(Exception): + """Raised when worker is not running.""" + pass + +def setup_database_background(data: dict) -> str: + global _active_setup_task_id, _last_error + + # Clear any previous error logs. + set_last_setup_error(None) + + if not is_worker_alive(): + set_last_setup_error("The Specify Worker is not running.") + raise MissingWorkerError("The Specify Worker is not running.") + + task_id = str(uuid4()) + logger.debug(f'task_id: {task_id}') + + args = [data] + + task = setup_database_task.apply_async(args, task_id=task_id) + + with _active_setup_lock: + _active_setup_task_id = task.id + + return task.id + +def is_worker_alive(): + """Pings the worker to see if its running.""" + try: + res = app.control.inspect(timeout=1).ping() + return bool(res) + except Exception: + return False + +def get_active_setup_task() -> Tuple[Optional[AsyncResult], bool]: + """Return the current setup task if it is active, and also if it is busy.""" + global _active_setup_task_id + with _active_setup_lock: + task_id = _active_setup_task_id + + if not task_id: + return None, False + + res = app.AsyncResult(task_id) + busy = res.state in ("PENDING", "RECEIVED", "STARTED", "RETRY", "PROGRESS") + # Clear the setup id if its not busy. + if not busy and res.state in ("SUCCESS", "FAILURE", "REVOKED"): + with _active_setup_lock: + if _active_setup_task_id == task_id: + _active_setup_task_id = None + return res, busy + +@app.task(bind=True) +def setup_database_task(self, data: dict): + """Execute all database setup steps in order.""" + self.update_state(state='STARTED', meta={'progress': api.get_setup_resource_progress()}) + def update_progress(): + self.update_state(state='STARTED', meta={'progress': api.get_setup_resource_progress()}) + + try: + with transaction.atomic(): + logger.debug('## SETTING UP DATABASE WITH SETTINGS:##') + logger.debug(data) + + logger.debug('Creating institution') + api.create_institution(data['institution']) + update_progress() + + logger.debug('Creating storage tree') + api.create_storage_tree(data['storagetreedef']) + update_progress() + + if data['institution'].get('issinglegeographytree', False) == True: + logger.debug('Creating singular geography tree') + api.create_global_geography_tree(data['globalgeographytreedef']) + + logger.debug('Creating division') + api.create_division(data['division']) + update_progress() + + discipline_type = data['discipline'].get('type', '') + is_paleo_geo = discipline_type in PALEO_DISCIPLINES or discipline_type in GEOLOGY_DISCIPLINES + default_tree = { + 'fullnamedirection': 1, + 'ranks': { + '0': True + } + } + + if is_paleo_geo: + logger.debug('Creating Chronostratigraphy tree') + api.create_geologictimeperiod_tree(default_tree.copy()) + + if data['institution'].get('issinglegeographytree', False) == False: + logger.debug('Creating geography tree') + api.create_geography_tree(data['geographytreedef']) + + logger.debug('Creating discipline') + api.create_discipline(data['discipline']) + update_progress() + + if is_paleo_geo: + logger.debug('Creating Lithostratigraphy tree') + api.create_lithostrat_tree(default_tree.copy()) + + logger.debug('Creating Tectonic Unit tree') + api.create_tectonicunit_tree(default_tree.copy()) + + logger.debug('Creating taxon tree') + api.create_taxon_tree(data['taxontreedef']) + update_progress() + + logger.debug('Creating collection') + api.create_collection(data['collection']) + update_progress() + + logger.debug('Creating specify user') + api.create_specifyuser(data['specifyuser']) + update_progress() + except Exception as e: + logger.exception(f'Error setting up database: {e}') + self.update_state(state='FAILURE', meta={'error': str(e)}) + raise + +def get_last_setup_error() -> Optional[str]: + global _last_error + return _last_error + +def set_last_setup_error(error_text: Optional[str]): + global _last_error + with _last_error_lock: + _last_error = error_text \ No newline at end of file diff --git a/specifyweb/backend/setup_tool/tree_defaults.py b/specifyweb/backend/setup_tool/tree_defaults.py new file mode 100644 index 00000000000..db27d297d5b --- /dev/null +++ b/specifyweb/backend/setup_tool/tree_defaults.py @@ -0,0 +1,61 @@ +from django.db import transaction +from django.db.models import Model as DjangoModel +from typing import Type + +from ..trees.utils import get_models + +import logging +logger = logging.getLogger(__name__) + +def create_default_tree(name: str, kwargs: dict, ranks: dict): + """Creates an initial empty tree. This should not be used outside of the database setup.""" + with transaction.atomic(): + tree_def_model, tree_rank_model, tree_node_model = get_models(name) + + if tree_def_model.objects.count() > 0: + raise RuntimeError(f'Tree {name} already exists, cannot create default.') + + # Create tree definition + treedef = tree_def_model.objects.create( + name=name, + **kwargs, + ) + + # Create tree ranks + treedefitems_bulk = [] + for rank_id, enabled in ranks.items(): + if enabled: + treedefitems_bulk.append( + tree_rank_model( + treedef=treedef, + name=str(rank_id), # TODO: allow rank name configuration + rankid=int(rank_id), + ) + ) + if treedefitems_bulk: + tree_rank_model.objects.bulk_create(treedefitems_bulk, ignore_conflicts=True) + + tree_def_item, create = tree_rank_model.objects.get_or_create( + treedef=treedef, + rankid=0 + ) + + # Create root node + # TODO: Avoid having duplicated code from add_root endpoint + root_node = tree_node_model.objects.create( + name="Root", + isaccepted=1, + nodenumber=1, + rankid=0, + parent=None, + definition=treedef, + definitionitem=tree_def_item, + fullname="Root" + ) + + return treedef + +def update_tree_scoping(treedef: Type[DjangoModel], discipline_id: int): + """Trees may be created before a discipline is created. This will update their discipline.""" + setattr(treedef, "discipline_id", discipline_id) + treedef.save(update_fields=["discipline_id"]) \ No newline at end of file diff --git a/specifyweb/backend/setup_tool/urls.py b/specifyweb/backend/setup_tool/urls.py new file mode 100644 index 00000000000..3858364b4ef --- /dev/null +++ b/specifyweb/backend/setup_tool/urls.py @@ -0,0 +1,21 @@ + +from django.urls import re_path + +from . import views + +urlpatterns = [ + # check if the db is new at login + re_path(r'^setup_progress/$', views.get_setup_progress), + + re_path(r'^setup_database/create/$', views.setup_database_view), + + re_path(r'^institution/create/$', views.create_institution_view), + re_path(r'^storagetreedef/create/$', views.create_storage_tree_view), + re_path(r'^global_geographytreedef/create/$', views.create_geography_tree_view), + re_path(f'^division/create/$', views.create_division_view), + re_path(f'^discipline/create/$', views.create_discipline_view), + re_path(f'^geographytreedef/create/$', views.create_geography_tree_view), + re_path(f'^taxontreedef/create/$', views.create_taxon_tree_view), + re_path(f'^collection/create/$', views.create_collection_view), + re_path(f'^specifyuser/create/$', views.create_specifyuser_view), +] \ No newline at end of file diff --git a/specifyweb/backend/setup_tool/utils.py b/specifyweb/backend/setup_tool/utils.py new file mode 100644 index 00000000000..4ba188ed66b --- /dev/null +++ b/specifyweb/backend/setup_tool/utils.py @@ -0,0 +1,51 @@ +import json +from pathlib import Path +from django.db.models import Model as DjangoModel +from typing import Optional, Type +from specifyweb.specify.api_utils import strict_uri_to_model + +import logging +logger = logging.getLogger(__name__) + +def resolve_uri_or_fallback(uri: Optional[str], id: Optional[int], table: Type[DjangoModel]) -> Optional[DjangoModel]: + """Retrieves a record from a URI or ID, falling back to the last created record if it exists.""" + if uri is not None: + # Try to resolve uri. It must be valid. + try: + uri_table, uri_id = strict_uri_to_model(uri, table._meta.db_table) + return table.objects.filter(pk=uri_id).first() + except Exception as e: + raise ValueError(e) + elif id is not None: + # Try to use the provided id. It must be valid. + try: + return table.objects.get(pk=id) + except table.DoesNotExist: + raise table.DoesNotExist(f"{table.name} with id {id} not found") + # Fallback to last created record. + return table.objects.last() + +def load_json_from_file(path: Path): + """ + Read a JSON file included within Specify directories. The file is expected to exist. + """ + + if path.exists() and path.is_file(): + try: + with path.open('r', encoding='utf-8') as fh: + return json.load(fh) + except json.JSONDecodeError as e: + logger.exception('Failed to decode JSON from %s: %s', path, e) + return None + except Exception as e: + logger.exception('Failed to decode JSON from %s: %s', path, e) + return None + else: + logger.debug('JSON file at %s does not exist.', path) + return None + +def normalize_keys(obj): + if isinstance(obj, dict): + return {k.lower(): normalize_keys(v) for k, v in obj.items()} + else: + return obj \ No newline at end of file diff --git a/specifyweb/backend/setup_tool/views.py b/specifyweb/backend/setup_tool/views.py new file mode 100644 index 00000000000..d22efd8f4c2 --- /dev/null +++ b/specifyweb/backend/setup_tool/views.py @@ -0,0 +1,38 @@ +from django import http +import json + +from specifyweb.backend.setup_tool import api + +def setup_database_view(request): + return api.setup_database(request) + +def create_institution_view(request): + return api.handle_request(request, api.create_institution) + +def create_storage_tree_view(request): + return api.handle_request(request, api.create_storage_tree) + +def create_global_geography_tree_view(request): + return api.handle_request(request, api.create_global_geography_tree) + +def create_division_view(request): + return api.handle_request(request, api.create_division) + +def create_discipline_view(request): + return api.handle_request(request, api.create_discipline) + +def create_geography_tree_view(request): + return api.handle_request(request, api.create_geography_tree) + +def create_taxon_tree_view(request): + return api.handle_request(request, api.create_taxon_tree) + +def create_collection_view(request): + return api.handle_request(request, api.create_collection) + +def create_specifyuser_view(request): + return api.handle_request(request, api.create_specifyuser) + +# check which resource are present in a new db to define setup step +def get_setup_progress(request): + return http.JsonResponse(api.get_setup_progress()) \ No newline at end of file diff --git a/specifyweb/frontend/js_src/lib/components/InitialContext/systemInfo.ts b/specifyweb/frontend/js_src/lib/components/InitialContext/systemInfo.ts index da2414c3854..21518f41335 100644 --- a/specifyweb/frontend/js_src/lib/components/InitialContext/systemInfo.ts +++ b/specifyweb/frontend/js_src/lib/components/InitialContext/systemInfo.ts @@ -44,7 +44,7 @@ function buildStatsLambdaUrl(base: string | null | undefined): string | null { if (!hasRoute) { const stage = 'prod'; const route = 'AggrgatedSp7Stats'; - u = `${u.replace(/\/$/, '') }/${stage}/${route}`; + u = `${u.replace(/\/$/, '')}/${stage}/${route}`; } return u; } @@ -58,7 +58,10 @@ export const fetchContext = load( if (systemInfo.stats_url !== null) { let counts: StatsCounts | null = null; try { - counts = await load('/context/stats_counts.json', 'application/json'); + counts = await load( + '/context/stats_counts.json', + 'application/json' + ); } catch { // If counts fetch fails, proceed without them. counts = null; @@ -102,12 +105,13 @@ export const fetchContext = load( const lambdaUrl = buildStatsLambdaUrl(systemInfo.stats_2_url); if (lambdaUrl) { - await ping(formatUrl(lambdaUrl, parameters, false), { errorMode: 'silent' }) - .catch(softFail); + await ping(formatUrl(lambdaUrl, parameters, false), { + errorMode: 'silent', + }).catch(softFail); } } return systemInfo; }); -export const getSystemInfo = (): SystemInfo => systemInfo; \ No newline at end of file +export const getSystemInfo = (): SystemInfo => systemInfo; diff --git a/specifyweb/frontend/js_src/lib/components/Login/index.tsx b/specifyweb/frontend/js_src/lib/components/Login/index.tsx index 59f98d09be9..dd1a1a4271c 100644 --- a/specifyweb/frontend/js_src/lib/components/Login/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/Login/index.tsx @@ -19,24 +19,42 @@ import { Form, Input, Label } from '../Atoms/Form'; import { Submit } from '../Atoms/Submit'; import { LoadingContext } from '../Core/Contexts'; import { SplashScreen } from '../Core/SplashScreen'; +import { LoadingScreen } from '../Molecules/Dialog'; +import { SetupTool } from '../SetupTool'; import { handleLanguageChange, LanguageSelection } from '../Toolbar/Language'; import type { OicProvider } from './OicLogin'; import { OicLogin } from './OicLogin'; +export type SetupResources = { + readonly institution: boolean; + readonly storageTreeDef: boolean; + readonly globalGeographyTreeDef: boolean; + readonly division: boolean; + readonly discipline: boolean; + readonly geographyTreeDef: boolean; + readonly taxonTreeDef: boolean; + readonly collection: boolean; + readonly specifyUser: boolean; +} + +export type SetupProgress = { + readonly resources: SetupResources; + readonly busy: boolean; + readonly error?: string; +}; + export function Login(): JSX.Element { - const [isNewUser] = useAsyncState( + const [setupProgress, setSetupProgress] = useAsyncState( React.useCallback( async () => - ajax(`/api/specify/is_new_user/`, { + ajax(`/setup_tool/setup_progress/`, { method: 'GET', - headers: { - Accept: 'application/json', - }, + headers: { Accept: 'application/json' }, errorMode: 'silent', }) .then(({ data }) => data) .catch((error) => { - console.error('Failed to fetch isNewUser:', error); + console.error('Failed to fetch setup progress:', error); return undefined; }), [] @@ -48,9 +66,11 @@ export function Login(): JSX.Element { const nextUrl = parseDjangoDump('next-url') ?? '/specify/'; const providers = parseDjangoDump>('providers') ?? []; - if (isNewUser === true || isNewUser === undefined) { - // Display here the new setup pages - return

Welcome! No institutions are available at the moment.

; + console.log(setupProgress); + if (setupProgress === undefined) return ; + + if (setupProgress.busy || (setupProgress.hasOwnProperty('resources') && Object.values(setupProgress.resources).includes(false))) { + return ; } return providers.length > 0 ? ( @@ -85,7 +105,7 @@ export function Login(): JSX.Element { } /> ); - }, [isNewUser]); + }, [setupProgress]); } const nextDestination = '/accounts/choose_collection/?next='; diff --git a/specifyweb/frontend/js_src/lib/components/SetupTool/index.tsx b/specifyweb/frontend/js_src/lib/components/SetupTool/index.tsx new file mode 100644 index 00000000000..611cd00c9ed --- /dev/null +++ b/specifyweb/frontend/js_src/lib/components/SetupTool/index.tsx @@ -0,0 +1,569 @@ +import React from 'react'; +import type { LocalizedString } from 'typesafe-i18n'; + +import { useId } from '../../hooks/useId'; +import { commonText } from '../../localization/common'; +import { headerText } from '../../localization/header'; +import { setupToolText } from '../../localization/setupTool'; +import { userText } from '../../localization/user'; +import { ajax } from '../../utils/ajax'; +import { Http } from '../../utils/ajax/definitions'; +import { type RA, localized } from '../../utils/types'; +import { Container, H2, H3 } from '../Atoms'; +import { Progress } from '../Atoms'; +import { Button } from '../Atoms/Button'; +import { Form, Input, Label, Select } from '../Atoms/Form'; +import { dialogIcons } from '../Atoms/Icons'; +import { Link } from '../Atoms/Link'; +import { Submit } from '../Atoms/Submit'; +import { LoadingContext } from '../Core/Contexts'; +import type { SetupProgress, SetupResources } from '../Login'; +import { loadingBar } from '../Molecules'; +import { MIN_PASSWORD_LENGTH } from '../Security/SetPassword'; +import type { FieldConfig, ResourceConfig } from './setupResources'; +import { FIELD_MAX_LENGTH, resources } from './setupResources'; + +type ResourceFormData = Record; + +const stepOrder: RA = [ + 'institution', + 'storageTreeDef', + 'globalGeographyTreeDef', + 'division', + 'discipline', + 'geographyTreeDef', + 'taxonTreeDef', + 'collection', + 'specifyUser', +]; + +function findNextStep( + currentStep: number, + formData: ResourceFormData, + direction: number = 1 +): number { + /* + * Find the next *accessible* form. + * Handles conditional pages, like the global geography tree. + */ + let step = currentStep + direction; + while (step >= 0 && step < resources.length) { + const resource = resources[step]; + if (resource.condition === undefined) { + return step; + } + // Check condition + let pass = true; + for (const [resourceName, fields] of Object.entries(resource.condition)) { + for (const [fieldName, requiredValue] of Object.entries(fields)) { + if (formData[resourceName][fieldName] !== requiredValue) { + pass = false; + break; + } + } + if (!pass) break; + } + if (pass) return step; + step += direction; + } + return currentStep; +} + +function getFormValue( + formData: ResourceFormData, + currentStep: number, + fieldName: string +): number | string | undefined { + return formData[resources[currentStep].resourceName][fieldName]; +} + +function useFormDefaults( + resource: ResourceConfig, + setFormData: (data: ResourceFormData) => void, + currentStep: number +): void { + const resourceName = resources[currentStep].resourceName; + const defaultFormData: ResourceFormData = {}; + const applyFieldDefaults = (field: FieldConfig, parentName?: string) => { + const fieldName = + parentName === undefined ? field.name : `${parentName}.${field.name}`; + if (field.type === 'object' && field.fields !== undefined) + field.fields.forEach((field) => applyFieldDefaults(field, fieldName)); + if (field.default !== undefined) defaultFormData[fieldName] = field.default; + }; + resource.fields.forEach((field) => applyFieldDefaults(field)); + setFormData((previous: any) => ({ + ...previous, + [resourceName]: { + ...defaultFormData, + ...previous[resourceName], + }, + })); +} + +export function SetupTool({ + setupProgress, + setSetupProgress, +}: { + readonly setupProgress: SetupProgress; + readonly setSetupProgress: ( + value: + | SetupProgress + | ((oldValue: SetupProgress | undefined) => SetupProgress | undefined) + | undefined + ) => void; +}): JSX.Element { + const formRef = React.useRef(null); + const [formData, setFormData] = React.useState( + Object.fromEntries(stepOrder.map((key) => [key, {}])) + ); + const [temporaryFormData, setTemporaryFormData] = + React.useState({}); // For front-end only. + + const [currentStep, setCurrentStep] = React.useState(0); + React.useEffect(() => { + useFormDefaults(resources[currentStep], setFormData, currentStep); + }, [currentStep]); + + // Keep track of the last backend error. + const [setupError, setSetupError] = React.useState( + undefined + ); + + // Is the database currrently being created? + const [inProgress, setInProgress] = React.useState(false); + const nextIncompleteStep = stepOrder.findIndex( + (resourceName) => !setupProgress.resources[resourceName] + ); + React.useEffect(() => { + if (setupProgress.busy) { + setInProgress(true); + } + }, [setupProgress]); + React.useEffect(() => { + // Poll for the latest setup progress. + if (!inProgress) return; + + const interval = setInterval( + async () => + ajax(`/setup_tool/setup_progress/`, { + method: 'GET', + headers: { Accept: 'application/json' }, + errorMode: 'dismissible', + }) + .then(({ data }) => { + setSetupProgress(data); + if (data.error !== undefined) setSetupError(data.error); + }) + .catch((error) => { + console.error('Failed to fetch setup progress:', error); + return undefined; + }), + 3000 + ); + + return () => clearInterval(interval); + }, [inProgress, setSetupProgress]); + + const loading = React.useContext(LoadingContext); + + const onResourceSaved = async ( + endpoint: string, + resourceLabel: string, + data: ResourceFormData + ): Promise => + ajax(endpoint, { + method: 'POST', + headers: { + Accept: 'application/json', + 'Content-Type': 'application/json', + }, + body: JSON.stringify(flattenAllResources(data)), + errorMode: 'visible', + expectedErrors: [Http.CREATED], + }) + .then(({ data, status }) => { + if (status === Http.OK) { + console.log(`${resourceLabel} created successfully:`, data); + return data; + } else { + console.error(`Error creating ${resourceLabel}:`, data); + throw new Error(`Issue when creating ${resourceLabel}`); + } + }) + .catch((error) => { + console.log(error); + console.error(`Request failed for ${resourceLabel}:`, error); + setSetupError(String(error)); + throw error; + }); + + const handleChange = ( + name: string, + newValue: LocalizedString | boolean + ): void => { + setFormData((previous) => { + const resourceName = resources[currentStep].resourceName; + return { + ...previous, + [resourceName]: { + ...previous[resourceName], + [name]: newValue, + }, + }; + }); + }; + + const handleSubmit = (event: React.FormEvent): void => { + event.preventDefault(); + + if (currentStep === resources.length - 1) { + /* + * Send resources to backend to start the setup + * const { endpoint, resourceName } = resources[currentStep]; + */ + const endpoint = '/setup_tool/setup_database/create/'; + loading( + onResourceSaved(endpoint, 'TEMPORARY_LABEL', formData) + .then((data) => { + console.log(data); + setSetupProgress(data.setup_progress as SetupProgress); + setInProgress(true); + }) + .catch((error) => { + console.error('Form submission failed:', error); + }) + ); + } else { + // Continue onto the next resource/form + setCurrentStep(findNextStep(currentStep, formData, 1)); + } + }; + + const handleBack = (): void => { + setCurrentStep(findNextStep(currentStep, formData, -1)); + }; + + const renderFormField = (field: FieldConfig, parentName?: string) => { + const { + name, + label, + type, + required = false, + description, + options, + fields, + passwordRepeat, + } = field; + + const fieldName = parentName === undefined ? name : `${parentName}.${name}`; + + const colSpan = type === 'object' ? 2 : 1; + return ( +
+ {type === 'boolean' ? ( +
+ + + handleChange(fieldName, isChecked) + } + /> + {label} + +
+ ) : type === 'select' && Array.isArray(options) ? ( +
+ + {label} + + +
+ ) : type === 'password' ? ( + <> + + {label} + { + handleChange(fieldName, value); + if (passwordRepeat !== undefined && formRef.current) { + const target = formRef.current.elements.namedItem( + passwordRepeat.name + ) as HTMLInputElement | null; + if (target) { + target.setCustomValidity( + target.value && target.value === value + ? '' + : userText.passwordsDoNotMatchError() + ); + } + } + }} + /> + + {passwordRepeat === undefined ? undefined : ( + + {passwordRepeat.label} + { + target.setCustomValidity( + target.value === + getFormValue(formData, currentStep, fieldName) + ? '' + : userText.passwordsDoNotMatchError() + ); + }} + onValueChange={(value) => + setTemporaryFormData((previous) => ({ + ...previous, + [passwordRepeat.name]: value, + })) + } + /> + + )} + + ) : type === 'object' ? ( + // Subforms +
+

+ {label} +

+ {fields === undefined ? undefined : renderFormFields(fields, name)} +
+ ) : ( + + {label} + handleChange(fieldName, value)} + /> + + )} +
+ ); + }; + + const renderFormFields = (fields: RA, parentName?: string) => ( +
+ {fields.map((field) => renderFormField(field, parentName))} +
+ ); + + const id = useId('setup-tool'); + + return ( + + +

+ {setupToolText.specifyConfigurationSetup()} +

+ {inProgress ? ( + +

+ {setupToolText.settingUp()} +

+

+ {nextIncompleteStep === -1 + ? setupToolText.settingUp() + : resources[nextIncompleteStep].label} +

+ {loadingBar} + + ) : ( +
+
+ +

+ {setupToolText.overview()} +

+
+ +
+ +
+
+ +
+
+

+ {resources[currentStep].label} +

+ {resources[currentStep].documentationUrl !== undefined && ( + + {headerText.documentation()} + + )} +
+ {resources[currentStep].description === + undefined ? undefined : ( +

+ {resources[currentStep].description} +

+ )} + {renderFormFields(resources[currentStep].fields)} +
+
+ + {commonText.back()} + + + {setupToolText.saveAndContinue()} + +
+ + + + + {setupError === undefined ? undefined : ( + +
+ {dialogIcons.warning} +

+ {setupToolText.setupError()} +

+
+

{localized(setupError)}

+ + )} +
+
+ )} + + ); +} + +function SetupOverview({ + formData, + currentStep, +}: { + readonly formData: ResourceFormData; + readonly currentStep: number; +}): JSX.Element { + // Display all previously filled out forms. + return ( +
+ + + + + + + {resources.map((resource, step) => { + // Display only the forms that have been visited. + if ( + Object.keys(formData[resource.resourceName]).length > 0 || + step <= currentStep + ) { + return ( + + + + + {resource.fields.map((field) => { + let value = + formData[resource.resourceName]?.[ + field.name + ]?.toString() ?? '-'; + if (field.type === 'object') { + value = '●'; + } else if (field.type === 'password') { + value = formData[resource.resourceName]?.[field.name] + ? '***' + : '-'; + } else if ( + field.type === 'select' && + Array.isArray(field.options) + ) { + const match = field.options.find( + (option) => String(option.value) === value + ); + value = match ? (match.label ?? match.value) : value; + } + return ( + + + + + ); + })} + + ); + } + return undefined; + })} + +
+ {resource.label} +
{field.label}{value}
+
+ ); +} + +// Turn 'table.field' keys to nested objects to send to the backend +function flattenToNested(data: Record): Record { + const result: Record = {}; + Object.entries(data).forEach(([key, value]) => { + if (key.includes('.')) { + const [prefix, field] = key.split('.', 2); + result[prefix] ||= {}; + result[prefix][field] = value; + } else { + result[key] = value; + } + }); + return result; +} +function flattenAllResources(data: Record): Record { + const result: Record = {}; + Object.entries(data).forEach(([key, value]) => { + result[key] = flattenToNested(value); + }); + return result; +} diff --git a/specifyweb/frontend/js_src/lib/components/SetupTool/setupResources.ts b/specifyweb/frontend/js_src/lib/components/SetupTool/setupResources.ts new file mode 100644 index 00000000000..4979613d589 --- /dev/null +++ b/specifyweb/frontend/js_src/lib/components/SetupTool/setupResources.ts @@ -0,0 +1,380 @@ +import type { LocalizedString } from 'typesafe-i18n'; + +import { setupToolText } from '../../localization/setupTool'; +import type { RA } from '../../utils/types'; + +// Default for max field length. +export const FIELD_MAX_LENGTH = 64; + +export type ResourceConfig = { + readonly resourceName: string; + readonly label: LocalizedString; + readonly endpoint: string; + readonly description?: LocalizedString; + readonly condition?: Record< + string, + Record + >; + readonly documentationUrl?: string; + readonly fields: RA; +}; + +type Option = { + readonly value: number | string; + readonly label?: string; +}; + +export type FieldConfig = { + readonly name: string; + readonly label: string; + readonly type?: 'boolean' | 'object' | 'password' | 'select' | 'text'; + readonly required?: boolean; + readonly default?: boolean | number | string; + readonly description?: string; + readonly options?: RA