Progress on validation

.dockerignore

@@ -0,0 +1,3 @@
+models/
+docker-volumes/
+

.gitignore

@@ -1,9 +1,25 @@
 __pycache__/
 *.py[cod]
 *.ifc
+*.json
 *.cover
 docker-volumes
 application/static/bimsurfer
 application/nix
 application/win
 application/ifc-pipeline.db
+application/ifc-python-parser
+application/ifc-parser
+application/bimsurfer_temp/
+application/static/test.jpg
+application/checks/ifcopenshell/
+application/checks/request_builder.py
+application/checks/dresult_bsdd.json
+application/checks/result_bsdd.json
+application/checks/__pycache__
+application/checks/.pytest_cache
+application/checks/tests/test_files
+application/decoded.json
+application/test_send.py
+*.txt
+application/ifcopenshell

.gitmodules

@@ -1,3 +1,9 @@
 [submodule "application/bimsurfer"]
 	path = application/bimsurfer
 	url = https://github.com/AECgeeks/BIMsurfer2
+[submodule "application/checks/ifc-python-parser"]
+	path = application/checks/ifc-python-parser
+	url = https://github.com/johltn/ifc-python-parser
+[submodule "application/checks/step-file-parser"]
+	path = application/checks/step-file-parser
+	url = https://github.com/IfcOpenShell/step-file-parser

README.md

@@ -9,3 +9,6 @@ There is a small web application in Flask that accepts file uploads. HTTPS is pr
 ./init.sh my.domain.name.com
 docker-compose up -d
 ~~~
+
+## Development notes
+Remember to store credentials as environment variables in `/etc/environment`

application/Dockerfile

@@ -1,8 +1,11 @@
 FROM debian:buster-slim
 
 WORKDIR /
-RUN apt-get -y update && apt-get -y --no-install-recommends --no-install-suggests install python3 python3-pip unzip wget libpq-dev build-essential libssl-dev libffi-dev libxml2-dev libxslt1-dev zlib1g-dev npm python3-setuptools python3-dev python3-wheel supervisor
-RUN python3 -m pip install flask flask-cors numpy Pillow gunicorn rq redis sqlalchemy psycopg2 sqlalchemy-utils Flask-BasicAuth flasgger requests flask-dropzone
+RUN apt-get -y update && apt-get -y --no-install-recommends --no-install-suggests install git python3 python3-pip unzip wget libpq-dev build-essential libssl-dev libffi-dev libxml2-dev libxslt1-dev zlib1g-dev npm python3-setuptools python3-dev python3-wheel supervisor
+
+RUN python3 -m pip install --upgrade pip
+
+RUN python3 -m pip install flask flask-cors numpy Pillow gunicorn rq redis SQLAlchemy==1.3.24 psycopg2 sqlalchemy-utils Flask-BasicAuth flasgger requests flask-dropzone lark-parser pyparsing XlsxWriter fuzzywuzzy python-Levenshtein requests_oauthlib Authlib requests argparse --upgrade
 RUN npm install -g jsdoc gltf-pipeline requirejs npx uglify-js
 
 # IfcConvert v0.6.0
@@ -13,6 +16,7 @@ RUN wget -O /tmp/ifcopenshell_python.zip https://s3.amazonaws.com/ifcopenshell-b
 RUN mkdir -p `python3 -c 'import site; print(site.getusersitepackages())'`
 RUN unzip -d `python3 -c 'import site; print(site.getusersitepackages())'` /tmp/ifcopenshell_python.zip
 
+
 # Temporary 'hotfix'
 RUN wget  -O `python3 -c 'import site; print(site.getusersitepackages())'`/ifcopenshell/validate.py https://raw.githubusercontent.com/IfcOpenShell/IfcOpenShell/v0.6.0/src/ifcopenshell-python/ifcopenshell/validate.py
 
@@ -21,6 +25,18 @@ WORKDIR /www
 COPY application/*.py /www/
 COPY application/templates /www/templates
 
+COPY application/checks /www/checks
+RUN unzip -d  /www/checks /tmp/ifcopenshell_python.zip
+
+
+RUN mkdir -p /www/checks/ifcopenshell/mvd
+RUN echo 1
+RUN git clone https://github.com/opensourceBIM/python-mvdxml /www/checks/ifcopenshell/mvd
+
+# RUN ls www/checks/ifcopenshell/
+# RUN mv python-mvdxml/* www/checks/ifcopenshell/mvd
+# RUN ls www/checks/ifcopenshell/
+
 COPY .git/HEAD /tmp/.git/HEAD
 COPY .git/refs/ /tmp/.git/refs/
 RUN /bin/bash -c '(cat /tmp/.git/$(cat /tmp/.git/HEAD | cut -d \  -f 2)) || cat /tmp/.git/HEAD' > /version

application/checks/check_MVD.py

@@ -0,0 +1,58 @@
+import ifcopenshell
+from ifcopenshell.mvd import mvd
+import logging
+import json
+import sys
+import os
+import time
+
+def validate_mvd(mvd_fn):
+    mvd_concept_roots = ifcopenshell.mvd.concept_root.parse(mvd_fn)
+    passed = 1
+
+    for concept_root in mvd_concept_roots:
+        try: #todo: check mvdXML file schema   
+            entity_type = concept_root.entity
+            if len(ifc_file.by_type(entity_type)):
+                entity_instances = ifc_file.by_type(entity_type)
+                for concept in concept_root.concepts():
+                    for rule in concept.template().rules:
+
+                        for e in entity_instances:
+                            extraction = mvd.extract_data(rule,e)              
+                            for ex in extraction:
+                                print(ex)
+                                for k, v in ex.items():
+                                    print(v)
+                                    if v == "Nonexistent value":
+                                        passed = 0
+
+        except:
+            pass
+
+    return passed
+
+
+if __name__ == "__main__":
+    start_time = time.time()
+
+    ifc_fn = sys.argv[1]
+    ifc_file = ifcopenshell.open(ifc_fn)
+
+    mvd_fn = "./ifcopenshell/mvd/mvd_examples/officials/ReferenceView_V1-2.mvdxml"
+    mvd_fn= os.path.join(os.path.dirname(__file__), "ifcopenshell/mvd/mvd_examples/officials/ReferenceView_V1-2.mvdxml")
+    mvd_concept_roots = ifcopenshell.mvd.concept_root.parse(mvd_fn)
+
+    jsonresultout = os.path.join(os.getcwd(), "result_mvd.json")
+    passed = validate_mvd(mvd_fn)
+    print("--- %s seconds ---" % (time.time() - start_time))
+
+    if passed == 1:
+        mvd_result = {'mvd':'v'}
+    elif passed == 0:
+        mvd_result = {'mvd':'i'}
+
+
+    with open(jsonresultout, 'w', encoding='utf-8') as f:
+        json.dump(mvd_result, f, ensure_ascii=False, indent=4)
+

application/checks/check_bsdd_v2.py

@@ -0,0 +1,158 @@
+import ifcopenshell
+import sys 
+import requests
+import json
+import argparse
+from helper import database
+
+
+
+def get_classification_object(uri):
+    url = "https://bs-dd-api-prototype.azurewebsites.net/api/Classification/v3"
+    return requests.get(url, {'namespaceUri':uri})
+
+def validate_ifc_classification_reference(relating_classification):
+    uri = relating_classification.Location
+    bsdd_response = get_classification_object(uri)
+    if bsdd_response.status_code != 200:
+        return 0
+    elif bsdd_response.status_code == 200:
+        return bsdd_response
+
+def has_specifications(bsdd_response_content):
+    if bsdd_response_content["classificationProperties"]:
+        return 1
+    else:
+        return 0
+
+def validate_instance(constraint,ifc_file, instance):
+
+    result = {"pset_name":"pset not found","property_name":"pset not found","value":"pset not found","datatype":"pset not found" }
+    constraint = {
+        "specified_pset_name":constraint["propertySet"],
+        "specified_property_name" : constraint["name"],
+        "specified_datatype" : constraint["dataType"],
+        "specified_predefined_value" : constraint["predefinedValue"],
+    }
+
+
+    # Integrate these:
+    #   "maxExclusive": 0
+    #   "maxInclusive": 0
+    #   "minExclusive": 0
+    #   "minInclusive": 0
+    #   "pattern": ""
+
+    for definition in instance.IsDefinedBy:
+        if definition.is_a() == "IfcRelDefinesByProperties":
+
+            pset = definition.RelatingPropertyDefinition
+            if pset.Name == constraint["specified_pset_name"]:
+                result["property_name"] = "property not found"
+                result["value"] = "property not found"
+                result["datatype"] = "property not found"
+
+                result = {"pset_name":pset.Name,"property_name":"pset not found","value":"pset not found","datatype":"pset not found" }
+                for property in pset.HasProperties:
+                    if property.Name == constraint["specified_property_name"]:
+                        result["property_name"] = property.Name
+
+                        if isinstance(property.NominalValue, ifcopenshell.entity_instance):
+                            result["value"] = property.NominalValue[0]
+                            result["datatype"] = type(property.NominalValue[0])
+                        else:
+                            result["value"] = property.NominalValue
+                            result["datatype"] = type(property.NominalValue[0])
+
+
+
+    return {"constraint":constraint,"result":result}
+
+
+
+def check_bsdd(ifc_fn, task_id):
+
+    file_code = ifc_fn.split(".ifc")[0]
+    ifc_file = ifcopenshell.open(ifc_fn)
+
+    with database.Session() as session:
+        model = session.query(database.model).filter(database.model.code == file_code)[0]
+        file_id = model.id
+
+        n = len(ifc_file.by_type("IfcRelAssociatesClassification"))
+        if n:
+            percentages = [i * 100. / n for i in range(n+1)]
+            num_dots = [int(b) - int(a) for a, b in zip(percentages, percentages[1:])]
+
+        for idx, rel in enumerate(ifc_file.by_type("IfcRelAssociatesClassification")):
+
+            sys.stdout.write(num_dots[idx] * ".")
+            sys.stdout.flush()
+
+            related_objects = rel.RelatedObjects
+            relating_classification = rel.RelatingClassification
+
+            bsdd_response = validate_ifc_classification_reference(relating_classification)
+            bsdd_content = json.loads(bsdd_response.text)
+
+            for ifc_instance in related_objects:
+                instance = database.ifc_instance(ifc_instance.GlobalId, ifc_instance.is_a(), file_id)
+                session.add(instance)
+                session.flush()
+                instance_id = instance.id
+                session.commit()               
+
+                if bsdd_response:
+                    if has_specifications(bsdd_content):
+                        specifications = bsdd_content["classificationProperties"]
+                        for constraint in specifications: 
+                            bsdd_result = database.bsdd_result(task_id)
+                            # Should create instance entry
+                            bsdd_result.instance_id = instance_id
+
+                            bsdd_result.bsdd_classification_uri = bsdd_content["namespaceUri"]
+                            bsdd_result.bsdd_type_constraint = ";".join(bsdd_content["relatedIfcEntityNames"])
+                            bsdd_result.bsdd_property_constraint = json.dumps(constraint)
+                            bsdd_result.bsdd_property_uri = constraint["propertyNamespaceUri"]
+
+                            results = validate_instance(constraint, ifc_file, ifc_instance)["result"]
+
+                            bsdd_result.ifc_property_set = results["pset_name"]
+                            bsdd_result.ifc_property_name = results["property_name"]
+
+                            if not isinstance(results["datatype"], str):
+                                bsdd_result.ifc_property_type = results["datatype"].__name__
+                            bsdd_result.ifc_property_value = results["value"]
+
+                            session.add(bsdd_result)
+                            session.commit()
+                    else:
+                        # Record NULL in other fields
+                        bsdd_result = database.bsdd_result(task_id)
+                        bsdd_result.bsdd_property_constraint = "no constraint"
+                        session.add(bsdd_result)
+                        session.commit()
+                else:
+                    # Record NULL everywhere in bsdd_result
+                    bsdd_result = database.bsdd_result(task_id)
+                    bsdd_result.bsdd_classification_uri = "classification not found"
+                    session.add(bsdd_result)
+                    session.commit()
+
+        #todo: implement scores that actually validate or not the model
+        model = session.query(database.model).filter(database.model.code == file_code)[0]
+        model.status_bsdd = 'v'
+        session.commit()
+
+if __name__=="__main__":
+        parser = argparse.ArgumentParser(description="Generate classified IFC file")
+        parser.add_argument("--input","-i", default="Duplex_A_20110505.ifc", type=str)
+        parser.add_argument("--task","-t", default=0, type=int)
+
+        args = parser.parse_args()
+        check_bsdd(args.input, args.task)
+
+
+
+
+

application/checks/helper.py

@@ -0,0 +1,5 @@
+import os, sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
+
+import database

application/checks/info.py

@@ -0,0 +1,33 @@
+import sys, os
+import ifcopenshell
+from helper import database
+
+ifc_fn = sys.argv[1]
+ifc_file = ifcopenshell.open(ifc_fn)
+
+try:
+    detected_mvd = ifc_file.header.file_description.description[0].split(" ", 1)[1]
+    detected_mvd = detected_mvd[1:-1]
+except:
+    detected_mvd = "no MVD detected"
+
+try:
+    authoring_app = ifc_file.by_type("IfcApplication")[0].ApplicationFullName
+except:
+    authoring_app = 'no authoring app detected'
+
+with database.Session() as session:
+    model = session.query(database.model).filter(database.model.code == ifc_fn[:-4]).all()[0]
+    model.size = str(round(os.path.getsize(ifc_fn)*10**-6)) + "MB"
+    model.schema = ifc_file.schema
+    model.authoring_application = authoring_app
+    model.mvd = detected_mvd
+    model.number_of_elements = len(ifc_file.by_type("IfcBuildingElement"))
+    model.number_of_geometries = len(ifc_file.by_type("IfcShapeRepresentation"))
+    model.number_of_properties = len(ifc_file.by_type("IfcProperty"))
+
+    session.commit()
+    session.close()
+
+
+

application/checks/tests/test_bsdd.py

@@ -0,0 +1,11 @@
+import pytest 
+import sys
+import os
+
+path = os.path.join(os.path.dirname(__file__), "..")
+sys.path.insert(1, path)
+
+from check_bSDD import get_domains
+
+def test_domains():
+    assert len(get_domains()) == 18

application/checks/tests/test_parser.py

@@ -0,0 +1,21 @@
+import pytest 
+import sys
+import os
+
+import ifcopenshell
+
+path = os.path.join(os.path.dirname(__file__), "../step-file-parser")
+sys.path.insert(1, path)
+
+from parse_file import *
+
+def test_parsing():
+    filepath = os.path.join(os.path.dirname(__file__), "test_files\\Duplex_A_20110505.ifc")
+    ifc_file = ifcopenshell.open(filepath)
+
+    f = open(filepath, "r")
+    text = f.read() 
+    tree = ifc_parser.parse(text)
+    entities = process_tree(tree)
+
+    assert entities[375]['attributes'][0] == ifc_file.by_id(375).GlobalId