From 280bba298651c98f49957773b05558b9fa209fb0 Mon Sep 17 00:00:00 2001 From: Geert Hesselink Date: Thu, 2 Mar 2023 19:21:58 -0600 Subject: [PATCH 01/16] add preprocessed_bsdd_data --- application/bsdd_utils.py | 94 +++++++++++++++++++++++++++++++++++++++ application/main.py | 25 +++++++++++ application/utils.py | 9 ++++ 3 files changed, 128 insertions(+) diff --git a/application/bsdd_utils.py b/application/bsdd_utils.py index 881931c4..615a74a1 100644 --- a/application/bsdd_utils.py +++ b/application/bsdd_utils.py @@ -43,3 +43,97 @@ def get_inst(instance_id): return hierarchical_bsdd_results +import database +import json +import functools + +from collections import defaultdict +from urllib.parse import urlparse + +import ifcopenshell +import ifcopenshell.template + +import utils + +def get_hierarchical_bsdd(id): + with database.Session() as session: + @functools.lru_cache(maxsize=128) + def get_inst(instance_id): + return session.query(database.ifc_instance).filter(database.ifc_instance.id == instance_id).all()[0] + + model = session.query(database.model).filter( + database.model.code == id).all()[0] + + bsdd_task = [task for task in model.tasks if task.task_type == "bsdd_validation_task"][0] + hierarchical_bsdd_results = {} + if model.status_bsdd != 'n': + for bsdd_result in bsdd_task.results: + + bsdd_result = bsdd_result.serialize() + + if bsdd_result["instance_id"]: + inst = get_inst(bsdd_result["instance_id"]) + bsdd_result['global_id'], bsdd_result['ifc_type'] = inst.global_id, inst.ifc_type + + if bsdd_result["bsdd_property_constraint"]: + # Quick fix to handle the case with no constraint + try: + bsdd_result["bsdd_property_constraint"] = json.loads( + bsdd_result["bsdd_property_constraint"]) + except: + bsdd_result["bsdd_property_constraint"] = 0 + else: + bsdd_result["bsdd_property_constraint"] = 0 + + if bsdd_result["domain_file"] not in hierarchical_bsdd_results.keys(): + hierarchical_bsdd_results[bsdd_result["domain_file"]]= {} + + if bsdd_result["classification_file"] not in hierarchical_bsdd_results[bsdd_result["domain_file"]].keys(): + hierarchical_bsdd_results[bsdd_result["domain_file"]][bsdd_result["classification_file"]] = [] + + hierarchical_bsdd_results[bsdd_result["domain_file"]][bsdd_result["classification_file"]].append(bsdd_result) + + return hierarchical_bsdd_results + +def bsdd_data_processing(bsdd_task, bsdd_results, session): + bsdd_instances = [bsdd_table(result.serialize(), session) for result in bsdd_task.results] + bsdd_data = defaultdict(lambda: {'valid': 0, 'invalid': 0, 'source': get_domain(bsdd_results)}) + + for item in bsdd_instances: + classification = item['classification'] + validity = item['validity'] + bsdd_data[classification][validity] += 1 + + bsdd_data = [{**{'classification': k}, **v} for k, v in bsdd_data.items()] + return bsdd_data + +def bsdd_table(bsdd_result, session): + inst = get_inst(session, bsdd_result['instance_id']) + observed_type = inst.ifc_type + required_type = bsdd_result['bsdd_type_constraint'] + validity = "valid" if utils.do_try(lambda: ifcopenshell.template.create(schema_identifier="IFC4X3").create_entity(observed_type).is_a(required_type), 'invalid') else 'invalid' + return {'classification': observed_type, 'validity': validity} + +def get_inst(session, instance_id): + return session.query(database.ifc_instance).filter(database.ifc_instance.id == instance_id).all()[0] + +def get_domain(bsdd_results): + domain_file = 'domain_file' + uri = 'bsdd_classification_uri' + domain_sources = [] + default = 'classification not found' + for result in bsdd_results: + bsdd_uri = result[uri] + if bsdd_uri == default: + domain_sources.append(bsdd_uri) + else: + parse = urlparse(bsdd_uri) + parsed_domain_file = ''.join(char for char in result[domain_file] if char.isalnum()).lower() + domain_part = [part for part in parse.path.split('/') if parsed_domain_file in part][0] + url = parse.scheme + '/' + parse.netloc + '/' + 'uri' + '/' + domain_part + '/' + domain_sources.append(url) + sources = list(filter(lambda x: x != default, domain_sources)) + return sources[0] if sources else default + +def bsdd_report_quantity(bsdd_task, item): + return sum(bool(bsdd_result.serialize().get(item)) for bsdd_result in bsdd_task.results) diff --git a/application/main.py b/application/main.py index 81028485..7553c81b 100644 --- a/application/main.py +++ b/application/main.py @@ -794,6 +794,31 @@ def get_model(fn): return response else: return send_file(path) + +@application.route('/preprocess_bsdd/', methods=['GET']) +@login_required +def preprocess_bsdd(user_data, id): + with database.Session() as session: + model = session.query(database.model).filter( + database.model.code == id).first() + bsdd_task = [task for task in model.tasks if task.task_type == "bsdd_validation_task"][0] + bsdd_results = [result.serialize() for result in bsdd_task.results] + + if model.status_bsdd != 'n': + preprocessed_bsdd_data = { + 'bSDD classification found': { + 'name': [r['classification_name'] for r in bsdd_results][0], + 'Release data': 'n.a.', + 'Organisation': 'BuildingSMART', + 'classification_count' : bsdd_utils.bsdd_report_quantity(bsdd_task, 'classification_code'), + 'properties_count': bsdd_utils.bsdd_report_quantity(bsdd_task, 'ifc_property_set'), + 'domain_source' : bsdd_utils.get_domain(bsdd_results) + }, + 'bSDD data': bsdd_utils.bsdd_data_processing(bsdd_task, bsdd_results, session) + } + + return jsonify(preprocessed_bsdd_data) + """ # Create a file called routes.py with the following diff --git a/application/utils.py b/application/utils.py index 1a0dd25f..f9db2b9a 100644 --- a/application/utils.py +++ b/application/utils.py @@ -78,3 +78,12 @@ def send_message(msg_content, user_email, html=None): "html":html, "subject": "Validation Service update", "text": msg_content}) + +def do_try(fn, default=None): + try: + return fn() + except: + import traceback + traceback.print_exc() + return default + From 9b888a08667c11f714d62f282073d8f395fc5098 Mon Sep 17 00:00:00 2001 From: Geert Hesselink Date: Thu, 2 Mar 2023 19:23:11 -0600 Subject: [PATCH 02/16] remove duplicate --- application/bsdd_utils.py | 45 --------------------------------------- 1 file changed, 45 deletions(-) diff --git a/application/bsdd_utils.py b/application/bsdd_utils.py index 615a74a1..392f1030 100644 --- a/application/bsdd_utils.py +++ b/application/bsdd_utils.py @@ -2,51 +2,6 @@ import json import functools -def get_hierarchical_bsdd(id): - with database.Session() as session: - @functools.lru_cache(maxsize=128) - def get_inst(instance_id): - return session.query(database.ifc_instance).filter(database.ifc_instance.id == instance_id).all()[0] - - model = session.query(database.model).filter( - database.model.code == id).all()[0] - - bsdd_task = [task for task in model.tasks if task.task_type == "bsdd_validation_task"][0] - hierarchical_bsdd_results = {} - if model.status_bsdd != 'n': - for bsdd_result in bsdd_task.results: - - bsdd_result = bsdd_result.serialize() - - if bsdd_result["instance_id"]: - inst = get_inst(bsdd_result["instance_id"]) - bsdd_result['global_id'], bsdd_result['ifc_type'] = inst.global_id, inst.ifc_type - - if bsdd_result["bsdd_property_constraint"]: - # Quick fix to handle the case with no constraint - try: - bsdd_result["bsdd_property_constraint"] = json.loads( - bsdd_result["bsdd_property_constraint"]) - except: - bsdd_result["bsdd_property_constraint"] = 0 - else: - bsdd_result["bsdd_property_constraint"] = 0 - - if bsdd_result["domain_file"] not in hierarchical_bsdd_results.keys(): - hierarchical_bsdd_results[bsdd_result["domain_file"]]= {} - - if bsdd_result["classification_file"] not in hierarchical_bsdd_results[bsdd_result["domain_file"]].keys(): - hierarchical_bsdd_results[bsdd_result["domain_file"]][bsdd_result["classification_file"]] = [] - - hierarchical_bsdd_results[bsdd_result["domain_file"]][bsdd_result["classification_file"]].append(bsdd_result) - - return hierarchical_bsdd_results - - -import database -import json -import functools - from collections import defaultdict from urllib.parse import urlparse From 65182b793a2dcc991e73df29664a313e68517262 Mon Sep 17 00:00:00 2001 From: Geert Hesselink <54070862+Ghesselink@users.noreply.github.com> Date: Fri, 3 Mar 2023 07:35:45 -0600 Subject: [PATCH 03/16] Update application/main.py Validate logged user Co-authored-by: Thomas Krijnen --- application/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/application/main.py b/application/main.py index 7553c81b..1de76abe 100644 --- a/application/main.py +++ b/application/main.py @@ -801,6 +801,8 @@ def preprocess_bsdd(user_data, id): with database.Session() as session: model = session.query(database.model).filter( database.model.code == id).first() + if model.user_id != user_data["sub"]: + abort(403) bsdd_task = [task for task in model.tasks if task.task_type == "bsdd_validation_task"][0] bsdd_results = [result.serialize() for result in bsdd_task.results] From a28a04c5ffbf3760c177b6337592491ec8fa187c Mon Sep 17 00:00:00 2001 From: Geert Hesselink <54070862+Ghesselink@users.noreply.github.com> Date: Fri, 3 Mar 2023 07:36:10 -0600 Subject: [PATCH 04/16] Update application/main.py update endpoint url Co-authored-by: Thomas Krijnen --- application/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/main.py b/application/main.py index 1de76abe..6dafd323 100644 --- a/application/main.py +++ b/application/main.py @@ -795,7 +795,7 @@ def get_model(fn): else: return send_file(path) -@application.route('/preprocess_bsdd/', methods=['GET']) +@application.route('/api/bsdd/statistics/', methods=['GET']) @login_required def preprocess_bsdd(user_data, id): with database.Session() as session: From ad6cc142d2f02f2b76e538e8a84085e664c411ec Mon Sep 17 00:00:00 2001 From: Geert Hesselink Date: Fri, 3 Mar 2023 17:30:36 -0600 Subject: [PATCH 05/16] consider variation of domain sources --- application/bsdd_utils.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/application/bsdd_utils.py b/application/bsdd_utils.py index 392f1030..4d688455 100644 --- a/application/bsdd_utils.py +++ b/application/bsdd_utils.py @@ -73,7 +73,6 @@ def get_inst(session, instance_id): return session.query(database.ifc_instance).filter(database.ifc_instance.id == instance_id).all()[0] def get_domain(bsdd_results): - domain_file = 'domain_file' uri = 'bsdd_classification_uri' domain_sources = [] default = 'classification not found' @@ -82,13 +81,9 @@ def get_domain(bsdd_results): if bsdd_uri == default: domain_sources.append(bsdd_uri) else: - parse = urlparse(bsdd_uri) - parsed_domain_file = ''.join(char for char in result[domain_file] if char.isalnum()).lower() - domain_part = [part for part in parse.path.split('/') if parsed_domain_file in part][0] - url = parse.scheme + '/' + parse.netloc + '/' + 'uri' + '/' + domain_part + '/' - domain_sources.append(url) + domain_sources.append(bsdd_uri.split("/class")[0]) sources = list(filter(lambda x: x != default, domain_sources)) - return sources[0] if sources else default + return {item: sources.count(item) for item in sources} if sources else default def bsdd_report_quantity(bsdd_task, item): return sum(bool(bsdd_result.serialize().get(item)) for bsdd_result in bsdd_task.results) From 9236a20399a5c6a11c7cf528059e675df3ecf00d Mon Sep 17 00:00:00 2001 From: Geert Hesselink Date: Fri, 3 Mar 2023 18:18:58 -0600 Subject: [PATCH 06/16] update allowed instane types --- application/bsdd_utils.py | 24 +++++++++++++++++++----- application/main.py | 3 +-- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/application/bsdd_utils.py b/application/bsdd_utils.py index 4d688455..8d691f98 100644 --- a/application/bsdd_utils.py +++ b/application/bsdd_utils.py @@ -50,8 +50,8 @@ def get_inst(instance_id): return hierarchical_bsdd_results -def bsdd_data_processing(bsdd_task, bsdd_results, session): - bsdd_instances = [bsdd_table(result.serialize(), session) for result in bsdd_task.results] +def get_processed_bsdd_table(bsdd_task, bsdd_results, session, schema): + bsdd_instances = [bsdd_table(result.serialize(), session, schema) for result in bsdd_task.results] bsdd_data = defaultdict(lambda: {'valid': 0, 'invalid': 0, 'source': get_domain(bsdd_results)}) for item in bsdd_instances: @@ -62,12 +62,26 @@ def bsdd_data_processing(bsdd_task, bsdd_results, session): bsdd_data = [{**{'classification': k}, **v} for k, v in bsdd_data.items()] return bsdd_data -def bsdd_table(bsdd_result, session): +def instance_supertypes(observed_type, schema): + allowed_types = [observed_type] + + while True: + try: + result = (lambda x: ifcopenshell.ifcopenshell_wrapper.schema_by_name(schema).declaration_by_name(x).supertype().name())(result_list[-1]) + except AttributeError: #NoneType + break + + allowed_types.append(result) + return allowed_types + +def bsdd_table(bsdd_result, session, schema): inst = get_inst(session, bsdd_result['instance_id']) observed_type = inst.ifc_type required_type = bsdd_result['bsdd_type_constraint'] - validity = "valid" if utils.do_try(lambda: ifcopenshell.template.create(schema_identifier="IFC4X3").create_entity(observed_type).is_a(required_type), 'invalid') else 'invalid' - return {'classification': observed_type, 'validity': validity} + domain_source = bsdd_result['bsdd_classification_uri'] + validity = "valid" if required_type in instance_supertypes(observed_type, schema) else 'invalid' + + return {'classification': observed_type, 'validity': validity, 'domain_source': domain_source} def get_inst(session, instance_id): return session.query(database.ifc_instance).filter(database.ifc_instance.id == instance_id).all()[0] diff --git a/application/main.py b/application/main.py index 6dafd323..cbeff84b 100644 --- a/application/main.py +++ b/application/main.py @@ -816,12 +816,11 @@ def preprocess_bsdd(user_data, id): 'properties_count': bsdd_utils.bsdd_report_quantity(bsdd_task, 'ifc_property_set'), 'domain_source' : bsdd_utils.get_domain(bsdd_results) }, - 'bSDD data': bsdd_utils.bsdd_data_processing(bsdd_task, bsdd_results, session) + 'bSDD data': bsdd_utils.get_processed_bsdd_table(bsdd_task, bsdd_results, session, model.schema) } return jsonify(preprocessed_bsdd_data) - """ # Create a file called routes.py with the following # example content to add application-specific routes From 7906332b38088235de73b9af2d2ce6e06d0da9ab Mon Sep 17 00:00:00 2001 From: Geert Hesselink Date: Fri, 3 Mar 2023 18:42:10 -0600 Subject: [PATCH 07/16] add allowed_types to lambda func --- application/bsdd_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/bsdd_utils.py b/application/bsdd_utils.py index 8d691f98..4cb7c8bf 100644 --- a/application/bsdd_utils.py +++ b/application/bsdd_utils.py @@ -67,7 +67,7 @@ def instance_supertypes(observed_type, schema): while True: try: - result = (lambda x: ifcopenshell.ifcopenshell_wrapper.schema_by_name(schema).declaration_by_name(x).supertype().name())(result_list[-1]) + result = (lambda x: ifcopenshell.ifcopenshell_wrapper.schema_by_name(schema).declaration_by_name(x).supertype().name())(allowed_types[-1]) except AttributeError: #NoneType break From f4fdee4c3922a687bc0f3c990b52ece52e66e7af Mon Sep 17 00:00:00 2001 From: Geert Hesselink Date: Sat, 4 Mar 2023 07:15:37 -0600 Subject: [PATCH 08/16] consider variation of classification names --- application/bsdd_utils.py | 5 +++++ application/main.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/application/bsdd_utils.py b/application/bsdd_utils.py index 4cb7c8bf..d68b0d01 100644 --- a/application/bsdd_utils.py +++ b/application/bsdd_utils.py @@ -74,6 +74,11 @@ def instance_supertypes(observed_type, schema): allowed_types.append(result) return allowed_types +def get_classification_name(bsdd_results): + default = 'name not found' + names = list(filter(lambda x: x != default, [r['classification_name'] for r in bsdd_results])) + return {item: names.count(item) for item in names} if names else default + def bsdd_table(bsdd_result, session, schema): inst = get_inst(session, bsdd_result['instance_id']) observed_type = inst.ifc_type diff --git a/application/main.py b/application/main.py index cbeff84b..8837af2a 100644 --- a/application/main.py +++ b/application/main.py @@ -809,7 +809,7 @@ def preprocess_bsdd(user_data, id): if model.status_bsdd != 'n': preprocessed_bsdd_data = { 'bSDD classification found': { - 'name': [r['classification_name'] for r in bsdd_results][0], + 'name' : bsdd_utils.get_classification_name(bsdd_results), 'Release data': 'n.a.', 'Organisation': 'BuildingSMART', 'classification_count' : bsdd_utils.bsdd_report_quantity(bsdd_task, 'classification_code'), From b1610016bcf5115d7d43067c450df0fdc5e5d0ce Mon Sep 17 00:00:00 2001 From: Geert Hesselink Date: Sat, 4 Mar 2023 07:16:31 -0600 Subject: [PATCH 09/16] remove hardcoded unknown values --- application/main.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/application/main.py b/application/main.py index 8837af2a..59419c9e 100644 --- a/application/main.py +++ b/application/main.py @@ -810,8 +810,6 @@ def preprocess_bsdd(user_data, id): preprocessed_bsdd_data = { 'bSDD classification found': { 'name' : bsdd_utils.get_classification_name(bsdd_results), - 'Release data': 'n.a.', - 'Organisation': 'BuildingSMART', 'classification_count' : bsdd_utils.bsdd_report_quantity(bsdd_task, 'classification_code'), 'properties_count': bsdd_utils.bsdd_report_quantity(bsdd_task, 'ifc_property_set'), 'domain_source' : bsdd_utils.get_domain(bsdd_results) From 14265e6744a93f1243891a31b46697f02e9eab72 Mon Sep 17 00:00:00 2001 From: Geert Hesselink Date: Sat, 4 Mar 2023 07:21:01 -0600 Subject: [PATCH 10/16] leave comment bsdd_table functionality --- application/bsdd_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/application/bsdd_utils.py b/application/bsdd_utils.py index d68b0d01..057ee24e 100644 --- a/application/bsdd_utils.py +++ b/application/bsdd_utils.py @@ -51,6 +51,9 @@ def get_inst(instance_id): return hierarchical_bsdd_results def get_processed_bsdd_table(bsdd_task, bsdd_results, session, schema): + """ + Return data to use in 'bsdd data' table in file metrics report + """ bsdd_instances = [bsdd_table(result.serialize(), session, schema) for result in bsdd_task.results] bsdd_data = defaultdict(lambda: {'valid': 0, 'invalid': 0, 'source': get_domain(bsdd_results)}) From 5f5d70eab82e6668745c997f25fae974f58865c5 Mon Sep 17 00:00:00 2001 From: Geert Hesselink Date: Sat, 4 Mar 2023 07:24:19 -0600 Subject: [PATCH 11/16] update comment --- application/bsdd_utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/application/bsdd_utils.py b/application/bsdd_utils.py index 057ee24e..06f86c18 100644 --- a/application/bsdd_utils.py +++ b/application/bsdd_utils.py @@ -52,7 +52,9 @@ def get_inst(instance_id): def get_processed_bsdd_table(bsdd_task, bsdd_results, session, schema): """ - Return data to use in 'bsdd data' table in file metrics report + The function returns data for the 'bsdd data' table in a file metrics report. + More specifically,it returns a list of dictionaries containing the classifications (ifc instances, in this case) + and their valid/invalid counts. """ bsdd_instances = [bsdd_table(result.serialize(), session, schema) for result in bsdd_task.results] bsdd_data = defaultdict(lambda: {'valid': 0, 'invalid': 0, 'source': get_domain(bsdd_results)}) From 5d3759a4493752129a9e7af670333b1d5296c3d9 Mon Sep 17 00:00:00 2001 From: Geert Hesselink Date: Sat, 4 Mar 2023 07:44:21 -0600 Subject: [PATCH 12/16] serialize results just once --- application/bsdd_utils.py | 20 ++++++++++---------- application/main.py | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/application/bsdd_utils.py b/application/bsdd_utils.py index 06f86c18..d7f20323 100644 --- a/application/bsdd_utils.py +++ b/application/bsdd_utils.py @@ -50,14 +50,14 @@ def get_inst(instance_id): return hierarchical_bsdd_results -def get_processed_bsdd_table(bsdd_task, bsdd_results, session, schema): +def get_processed_bsdd_table(bsdd_results, session, schema): """ The function returns data for the 'bsdd data' table in a file metrics report. More specifically,it returns a list of dictionaries containing the classifications (ifc instances, in this case) and their valid/invalid counts. """ - bsdd_instances = [bsdd_table(result.serialize(), session, schema) for result in bsdd_task.results] - bsdd_data = defaultdict(lambda: {'valid': 0, 'invalid': 0, 'source': get_domain(bsdd_results)}) + bsdd_instances = [bsdd_table(result, session, schema) for result in bsdd_results] + bsdd_data = defaultdict(lambda: {'valid': 0, 'invalid': 0, 'domain_source': 'classification not found'}) for item in bsdd_instances: classification = item['classification'] @@ -79,11 +79,6 @@ def instance_supertypes(observed_type, schema): allowed_types.append(result) return allowed_types -def get_classification_name(bsdd_results): - default = 'name not found' - names = list(filter(lambda x: x != default, [r['classification_name'] for r in bsdd_results])) - return {item: names.count(item) for item in names} if names else default - def bsdd_table(bsdd_result, session, schema): inst = get_inst(session, bsdd_result['instance_id']) observed_type = inst.ifc_type @@ -96,6 +91,11 @@ def bsdd_table(bsdd_result, session, schema): def get_inst(session, instance_id): return session.query(database.ifc_instance).filter(database.ifc_instance.id == instance_id).all()[0] +def get_classification_name(bsdd_results): + default = 'name not found' + names = list(filter(lambda x: x != default, [r['classification_name'] for r in bsdd_results])) + return {item: names.count(item) for item in names} if names else default + def get_domain(bsdd_results): uri = 'bsdd_classification_uri' domain_sources = [] @@ -109,5 +109,5 @@ def get_domain(bsdd_results): sources = list(filter(lambda x: x != default, domain_sources)) return {item: sources.count(item) for item in sources} if sources else default -def bsdd_report_quantity(bsdd_task, item): - return sum(bool(bsdd_result.serialize().get(item)) for bsdd_result in bsdd_task.results) +def bsdd_report_quantity(bsdd_results, item): + return sum(bool(bsdd_result.get(item)) for bsdd_result in bsdd_results) diff --git a/application/main.py b/application/main.py index 59419c9e..85dccef9 100644 --- a/application/main.py +++ b/application/main.py @@ -810,11 +810,11 @@ def preprocess_bsdd(user_data, id): preprocessed_bsdd_data = { 'bSDD classification found': { 'name' : bsdd_utils.get_classification_name(bsdd_results), - 'classification_count' : bsdd_utils.bsdd_report_quantity(bsdd_task, 'classification_code'), - 'properties_count': bsdd_utils.bsdd_report_quantity(bsdd_task, 'ifc_property_set'), + 'classification_count' : bsdd_utils.bsdd_report_quantity(bsdd_results, 'classification_code'), + 'properties_count': bsdd_utils.bsdd_report_quantity(bsdd_results, 'ifc_property_set'), 'domain_source' : bsdd_utils.get_domain(bsdd_results) }, - 'bSDD data': bsdd_utils.get_processed_bsdd_table(bsdd_task, bsdd_results, session, model.schema) + 'bSDD data': bsdd_utils.get_processed_bsdd_table(bsdd_results, session, model.schema) } return jsonify(preprocessed_bsdd_data) From 816f747620f1fdadff8b22e74b8f1e5289bea707 Mon Sep 17 00:00:00 2001 From: Geert Hesselink Date: Sun, 5 Mar 2023 20:30:21 -0600 Subject: [PATCH 13/16] remove do_try func --- application/bsdd_utils.py | 2 -- application/utils.py | 9 --------- 2 files changed, 11 deletions(-) diff --git a/application/bsdd_utils.py b/application/bsdd_utils.py index d7f20323..70bc6f28 100644 --- a/application/bsdd_utils.py +++ b/application/bsdd_utils.py @@ -8,8 +8,6 @@ import ifcopenshell import ifcopenshell.template -import utils - def get_hierarchical_bsdd(id): with database.Session() as session: @functools.lru_cache(maxsize=128) diff --git a/application/utils.py b/application/utils.py index f9db2b9a..1a0dd25f 100644 --- a/application/utils.py +++ b/application/utils.py @@ -78,12 +78,3 @@ def send_message(msg_content, user_email, html=None): "html":html, "subject": "Validation Service update", "text": msg_content}) - -def do_try(fn, default=None): - try: - return fn() - except: - import traceback - traceback.print_exc() - return default - From b6a316e38e91bcb2c6602bce57312b58b8160239 Mon Sep 17 00:00:00 2001 From: Geert Hesselink Date: Sun, 5 Mar 2023 20:39:57 -0600 Subject: [PATCH 14/16] change endpoint name --- application/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/main.py b/application/main.py index 85dccef9..a2f2914e 100644 --- a/application/main.py +++ b/application/main.py @@ -795,7 +795,7 @@ def get_model(fn): else: return send_file(path) -@application.route('/api/bsdd/statistics/', methods=['GET']) +@application.route('/api/bsdd_statistics/', methods=['GET']) @login_required def preprocess_bsdd(user_data, id): with database.Session() as session: From 58a98f1f06ecb6a65d65104453be6857dc34b965 Mon Sep 17 00:00:00 2001 From: Geert Hesselink Date: Sun, 5 Mar 2023 21:37:59 -0600 Subject: [PATCH 15/16] duplicate get_domain function of bsdd_check_v2.py --- application/bsdd_utils.py | 24 +++++++++++++----------- application/main.py | 5 ++++- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/application/bsdd_utils.py b/application/bsdd_utils.py index 70bc6f28..858d409a 100644 --- a/application/bsdd_utils.py +++ b/application/bsdd_utils.py @@ -1,6 +1,7 @@ import database import json import functools +import collections from collections import defaultdict from urllib.parse import urlparse @@ -8,6 +9,8 @@ import ifcopenshell import ifcopenshell.template +import checks.check_bsdd_v2 + def get_hierarchical_bsdd(id): with database.Session() as session: @functools.lru_cache(maxsize=128) @@ -94,18 +97,17 @@ def get_classification_name(bsdd_results): names = list(filter(lambda x: x != default, [r['classification_name'] for r in bsdd_results])) return {item: names.count(item) for item in names} if names else default -def get_domain(bsdd_results): + +def domain_sources(bsdd_results): uri = 'bsdd_classification_uri' - domain_sources = [] - default = 'classification not found' - for result in bsdd_results: - bsdd_uri = result[uri] - if bsdd_uri == default: - domain_sources.append(bsdd_uri) - else: - domain_sources.append(bsdd_uri.split("/class")[0]) - sources = list(filter(lambda x: x != default, domain_sources)) - return {item: sources.count(item) for item in sources} if sources else default + domain_sources = [ + checks.check_bsdd_v2.get_domain(result[uri]).json()[0]['namespaceUri'] if result[uri] != 'classification not found' + else result[uri] + for result in bsdd_results + ] + sources = [source for source in domain_sources if source != 'classification not found'] + return collections.Counter(sources) if sources else 'classification not found' + def bsdd_report_quantity(bsdd_results, item): return sum(bool(bsdd_result.get(item)) for bsdd_result in bsdd_results) diff --git a/application/main.py b/application/main.py index a2f2914e..cd5afcda 100644 --- a/application/main.py +++ b/application/main.py @@ -29,6 +29,7 @@ import os import json import ast +import sys import threading from functools import wraps @@ -45,6 +46,8 @@ from requests_oauthlib import OAuth2Session from authlib.jose import jwt +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), 'checks'))) + import utils import bsdd_utils import database @@ -812,7 +815,7 @@ def preprocess_bsdd(user_data, id): 'name' : bsdd_utils.get_classification_name(bsdd_results), 'classification_count' : bsdd_utils.bsdd_report_quantity(bsdd_results, 'classification_code'), 'properties_count': bsdd_utils.bsdd_report_quantity(bsdd_results, 'ifc_property_set'), - 'domain_source' : bsdd_utils.get_domain(bsdd_results) + 'domain_source' : bsdd_utils.domain_sources(bsdd_results) }, 'bSDD data': bsdd_utils.get_processed_bsdd_table(bsdd_results, session, model.schema) } From 19deaf9e934d8e6e79445b9bf86657cf68aeba32 Mon Sep 17 00:00:00 2001 From: Geert Hesselink Date: Mon, 6 Mar 2023 08:36:38 -0600 Subject: [PATCH 16/16] modify check validity of bsdd classification --- application/bsdd_utils.py | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/application/bsdd_utils.py b/application/bsdd_utils.py index 858d409a..087ed045 100644 --- a/application/bsdd_utils.py +++ b/application/bsdd_utils.py @@ -68,24 +68,15 @@ def get_processed_bsdd_table(bsdd_results, session, schema): bsdd_data = [{**{'classification': k}, **v} for k, v in bsdd_data.items()] return bsdd_data -def instance_supertypes(observed_type, schema): - allowed_types = [observed_type] - - while True: - try: - result = (lambda x: ifcopenshell.ifcopenshell_wrapper.schema_by_name(schema).declaration_by_name(x).supertype().name())(allowed_types[-1]) - except AttributeError: #NoneType - break - - allowed_types.append(result) - return allowed_types - def bsdd_table(bsdd_result, session, schema): inst = get_inst(session, bsdd_result['instance_id']) observed_type = inst.ifc_type - required_type = bsdd_result['bsdd_type_constraint'] domain_source = bsdd_result['bsdd_classification_uri'] - validity = "valid" if required_type in instance_supertypes(observed_type, schema) else 'invalid' + + if all(bsdd_result[key] == 1 for key in ['val_ifc_type', 'val_property_set', 'val_property_name', 'val_property_type', 'val_property_value']): + validity = 'valid' + else: + validity = 'invalid' return {'classification': observed_type, 'validity': validity, 'domain_source': domain_source}