Merge branch 'master' into get-base-url-with-root-path

Author: ml-evs

.github/workflows/deps_lint.yml

@@ -141,7 +141,7 @@ jobs:
 
     services:
       mongo:
-        image: mongo:4.2
+        image: mongo:4
         ports:
         - 27017:27017
       postgres:
@@ -156,6 +156,14 @@ jobs:
           --health-retries 5
         ports:
         - 5432:5432
+      elasticsearch:
+        image: elasticsearch:6.8.13
+        ports:
+          - 9200:9200
+          - 9300:9300
+        env:
+          discovery.type: single-node
+
 
     steps:
     - uses: actions/checkout@v2
@@ -176,20 +184,29 @@ jobs:
         pip install -r requirements.txt
         pip install -r requirements-dev.txt
 
+    - name: Run all tests (using `mongomock`)
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml tests/
+      env:
+        OPTIMADE_DATABASE_BACKEND: 'mongomock'
+
+
     - name: Run server tests (using a real MongoDB)
-      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml tests/server
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server
       env:
-        OPTIMADE_CI_FORCE_MONGO: 1
+        OPTIMADE_DATABASE_BACKEND: 'mongodb'
 
-    - name: Run all tests (using `mongomock`)
-      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/
+    - name: Run server tests (using Elasticsearch)
+      run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/server
       env:
-        OPTIMADE_CI_FORCE_MONGO: 0
+        OPTIMADE_DATABASE_BACKEND: 'elastic'
 
     - name: Install adapter conversion dependencies
       run: |
         pip install -r requirements-client.txt
         # AiiDA-specific
+        # sqlalchemy install is temporary and has been fixed in aiidateam/aiida-core#4809,
+        # but is still not released in a new AiiDA version.
+        pip install sqlalchemy==1.3.23
         reentry scan
 
     - name: Setup up environment for AiiDA
@@ -200,9 +217,25 @@ jobs:
     - name: Run previously skipped tests for adapter conversion
       run: pytest -rs -vvv --cov=./optimade/ --cov-report=xml --cov-append tests/adapters/
 
-    - name: Run tests for validator only to assess coverage
+    - name: Run tests for validator only to assess coverage (mongomock)
       if: matrix.python-version == 3.8
-      run: pytest -rs --cov=./optimade/ --cov-report=xml:validator_cov.xml tests/server/test_server_validation.py
+      run: pytest -rs --cov=./optimade/ --cov-report=xml:validator_cov.xml --cov-append tests/server/test_server_validation.py
+      env:
+        OPTIMADE_DATABASE_BACKEND: 'mongomock'
+
+    - name: Run tests for validator only to assess coverage (Elasticsearch)
+      if: matrix.python-version == 3.8
+      run: pytest -rs --cov=./optimade/ --cov-report=xml:validator_cov.xml --cov-append tests/server/test_server_validation.py
+      env:
+        OPTIMADE_DATABASE_BACKEND: 'elastic'
+        OPTIMADE_INSERT_TEST_DATA: false # Must be specified as previous steps will have already inserted the test data
+
+    - name: Run tests for validator only to assess coverage (MongoDB)
+      if: matrix.python-version == 3.8
+      run: pytest -rs --cov=./optimade/ --cov-report=xml:validator_cov.xml --cov-append tests/server/test_server_validation.py
+      env:
+        OPTIMADE_DATABASE_BACKEND: 'mongodb'
+        OPTIMADE_INSERT_TEST_DATA: false # Must be specified as previous steps will have already inserted the test data
 
     - name: Upload coverage to Codecov
       if: matrix.python-version == 3.8 && github.repository == 'Materials-Consortia/optimade-python-tools'
@@ -238,10 +271,13 @@ jobs:
         pip install -U setuptools
         pip install -e .
         pip install -r requirements.txt
+        pip install -r requirements-dev.txt
         pip install -r requirements-docs.txt
 
     - name: Build
-      run: mkdocs build --strict --verbose
+      run: |
+        invoke create-api-reference-docs
+        mkdocs build --strict --verbose
 
   test_build:
     runs-on: ubuntu-latest

INSTALL.md

@@ -44,7 +44,7 @@ py.test
 # Install pre-commit environment (e.g., auto-formats code on `git commit`)
 pre-commit install
 
-# Optional: Install MongoDB (and set `use_real_mongo = true`)
+# Optional: Install MongoDB (and set `database_backend = mongodb`)
 # Below method installs in conda environment and
 # - starts server in background
 # - ensures and uses ~/dbdata directory to store data

README.md

@@ -11,10 +11,8 @@ The aim of OPTIMADE is to develop a common API, compliant with the [JSON API 1.0
 This is to enable interoperability among databases that contain calculated properties of existing and hypothetical materials.
 
 This repository contains a library of tools for implementing and consuming [OPTIMADE](https://www.optimade.org) APIs using Python.
-It also contains a server validator tool, which may be called from the shell or used as a GitHub Action from [optimade-validator-action](https://github.com/Materials-Consortia/optimade-validator-action).
-
-_Disclaimer_: While the package supports `elasticsearch-dsl` v6 & v7 and `django` v2 & v3, all tests are performed with the latest supported version.
-If you experience any issues with the older versions, you are most welcome to contribute to the repository (see below under [Contributing](#contributing)).
+Server implementations can make use of the supported MongoDB (v4) and Elasticsearch (v6) database backends, or plug in a custom backend implementation.
+The package also contains a server validator tool, which may be called from the shell (`optimade-validator`) or used as a GitHub Action from [optimade-validator-action](https://github.com/Materials-Consortia/optimade-validator-action).
 
 ## Status
 

docs/api_reference/server/entry_collections/elasticsearch.md

@@ -0,0 +1,3 @@
+# elasticsearch
+
+::: optimade.server.entry_collections.elasticsearch

docs/static/default_config.json

@@ -1,7 +1,7 @@
 {
     "config_file": "~/.optimade.json",
     "debug": false,
-    "use_real_mongo": false,
+    "insert_test_data": true,
     "mongo_database": "optimade",
     "mongo_uri": "localhost:27017",
     "links_collection": "links",

optimade/__init__.py

@@ -1,2 +1,21 @@
 __version__ = "0.13.3"
 __api_version__ = "1.0.1"
+
+
+import sys
+
+if sys.version_info.minor == 6:
+    # Python 3.6
+    import warnings
+
+    warnings.filterwarnings(
+        action="once",
+        message=r"v0\.14 of the `optimade` package.*",
+        category=DeprecationWarning,
+        append=False,
+    )
+    warnings.warn(
+        "v0.14 of the `optimade` package will be the last to support Python 3.6. "
+        "Please upgrade to Python 3.7+ to use v0.15 and later versions of `optimade`.",
+        DeprecationWarning,
+    )

optimade/filtertransformers/django.py

@@ -1,7 +1,7 @@
 import warnings
 
 warnings.warn(
-    "Django functionality is deprecated and will be removed in later versions (unless support is requested).",
+    "Django functionality is deprecated and will be removed in a later version (unless support is requested).",
     DeprecationWarning,
     stacklevel=2,
 )

optimade/filtertransformers/elasticsearch.py

@@ -2,7 +2,6 @@
 
 from lark import v_args
 from elasticsearch_dsl import Q, Text, Keyword, Integer, Field
-from optimade.models import CHEMICAL_SYMBOLS, ATOMIC_NUMBERS
 from optimade.filtertransformers import BaseTransformer
 from optimade.server.exceptions import BadRequest
 
@@ -14,7 +13,7 @@
 _has_operators = {"ALL": "must", "ANY": "should"}
 _length_quantities = {
     "elements": "nelements",
-    "elements_rations": "nelements",
+    "elements_ratios": "nelements",
     "dimension_types": "dimension_types",
 }
 
@@ -130,42 +129,50 @@ def _has_query_op(self, quantities, op, predicate_zip_list):
             # in elastic search. Only supported for elements, where we can construct
             # an anonymous "formula" based on elements sorted by order number and
             # can do a = comparision to check if all elements are contained
-            if len(quantities) > 1:
-                raise NotImplementedError("HAS ONLY is not supported with zip")
-            quantity = quantities[0]
 
-            if quantity.has_only_quantity is None:
-                raise NotImplementedError(
-                    "HAS ONLY is not supported by %s" % quantity.name
-                )
-
-            def values():
-                for predicates in predicate_zip_list:
-                    if len(predicates) != 1:
-                        raise NotImplementedError("Tuples not supported in HAS ONLY")
-                    op, value = predicates[0]
-                    if op != "=":
-                        raise NotImplementedError(
-                            "Predicated not supported in HAS ONLY"
-                        )
-                    if not isinstance(value, str):
-                        raise NotImplementedError("Only strings supported in HAS ONLY")
-                    yield value
-
-            try:
-                order_numbers = list([ATOMIC_NUMBERS[element] for element in values()])
-                order_numbers.sort()
-                value = "".join(
-                    [CHEMICAL_SYMBOLS[number - 1] for number in order_numbers]
-                )
-            except KeyError:
-                raise NotImplementedError(
-                    "HAS ONLY is only supported for chemical symbols"
-                )
+            # @ml-evs: Disabling this HAS ONLY workaround as tests are not passing
+            raise NotImplementedError(
+                "HAS ONLY queries are not currently supported by the Elasticsearch backend."
+            )
 
-            return Q("term", **{quantity.has_only_quantity.name: value})
+            # from optimade.models import CHEMICAL_SYMBOLS, ATOMIC_NUMBERS
+
+            # if len(quantities) > 1:
+            #     raise NotImplementedError("HAS ONLY is not supported with zip")
+            # quantity = quantities[0]
+
+            # if quantity.has_only_quantity is None:
+            #     raise NotImplementedError(
+            #         "HAS ONLY is not supported by %s" % quantity.name
+            #     )
+
+            # def values():
+            #     for predicates in predicate_zip_list:
+            #         if len(predicates) != 1:
+            #             raise NotImplementedError("Tuples not supported in HAS ONLY")
+            #         op, value = predicates[0]
+            #         if op != "=":
+            #             raise NotImplementedError(
+            #                 "Predicated not supported in HAS ONLY"
+            #             )
+            #         if not isinstance(value, str):
+            #             raise NotImplementedError("Only strings supported in HAS ONLY")
+            #         yield value
+
+            # try:
+            #     order_numbers = list([ATOMIC_NUMBERS[element] for element in values()])
+            #     order_numbers.sort()
+            #     value = "".join(
+            #         [CHEMICAL_SYMBOLS[number - 1] for number in order_numbers]
+            #     )
+            # except KeyError:
+            #     raise NotImplementedError(
+            #         "HAS ONLY is only supported for chemical symbols"
+            #     )
+
+            # return Q("term", **{quantity.has_only_quantity.name: value})
         else:
-            raise NotImplementedError
+            raise NotImplementedError(f"Unrecognised operation {op}.")
 
         queries = [
             self._has_query(quantities, predicates) for predicates in predicate_zip_list
@@ -320,12 +327,15 @@ def set_zip_op_rhs(self, args):
         return lambda quantity: self._has_query_op([quantity] + add_on, op, values)
 
     def property_zip_addon(self, args):
+        raise NotImplementedError("Correlated list queries are not supported.")
         return args
 
     def value_zip(self, args):
+        raise NotImplementedError("Correlated list queries are not supported.")
         return self.value_list(args)
 
     def value_zip_list(self, args):
+        raise NotImplementedError("Correlated list queries are not supported.")
         return args
 
     def value_list(self, args):

optimade/filtertransformers/mongo.py

@@ -61,11 +61,11 @@ def value_list(self, arg):
 
     def value_zip(self, arg):
         # value_zip: [ OPERATOR ] value ":" [ OPERATOR ] value (":" [ OPERATOR ] value)*
-        raise NotImplementedError
+        raise NotImplementedError("Correlated list queries are not supported.")
 
     def value_zip_list(self, arg):
         # value_zip_list: value_zip ( "," value_zip )*
-        raise NotImplementedError
+        raise NotImplementedError("Correlated list queries are not supported.")
 
     def expression(self, arg):
         # expression: expression_clause ( OR expression_clause )
@@ -158,11 +158,11 @@ def length_op_rhs(self, arg):
     def set_zip_op_rhs(self, arg):
         # set_zip_op_rhs: property_zip_addon HAS ( value_zip | ONLY value_zip_list | ALL value_zip_list |
         # ANY value_zip_list )
-        raise NotImplementedError
+        raise NotImplementedError("Correlated list queries are not supported.")
 
     def property_zip_addon(self, arg):
         # property_zip_addon: ":" property (":" property)*
-        raise NotImplementedError
+        raise NotImplementedError("Correlated list queries are not supported.")
 
     def _recursive_expression_phrase(self, arg):
         """Helper function for parsing `expression_phrase`. Recursively sorts out
@@ -226,15 +226,15 @@ def _apply_aliases(self, filter_: dict) -> dict:
             return filter_
 
         def check_for_alias(prop, expr):
-            return self.mapper.alias_for(prop) != prop
+            return self.mapper.get_backend_field(prop) != prop
 
         def apply_alias(subdict, prop, expr):
             if isinstance(subdict, dict):
-                subdict[self.mapper.alias_for(prop)] = self._apply_aliases(
+                subdict[self.mapper.get_backend_field(prop)] = self._apply_aliases(
                     subdict.pop(prop)
                 )
             elif isinstance(subdict, str):
-                subdict = self.mapper.alias_for(subdict)
+                subdict = self.mapper.get_backend_field(subdict)
 
             return subdict
 
@@ -396,7 +396,7 @@ def replace_str_id_with_objectid(subdict, prop, expr):
                 val = subdict[prop][operator]
                 if operator not in ("$eq", "$ne"):
                     if self.mapper is not None:
-                        prop = self.mapper.alias_of(prop)
+                        prop = self.mapper.get_optimade_field(prop)
                     raise NotImplementedError(
                         f"Operator {operator} not supported for query on field {prop!r}, can only test for equality"
                     )
@@ -417,7 +417,7 @@ def _apply_mongo_date_filter(self, filter_: dict) -> dict:
         def check_for_timestamp_field(prop, _):
             """ Find cases where the query dict is operating on a timestamp field. """
             if self.mapper is not None:
-                prop = self.mapper.alias_of(prop)
+                prop = self.mapper.get_optimade_field(prop)
             return prop == "last_modified"
 
         def replace_str_date_with_datetime(subdict, prop, expr):