Add feature not detected test result

framework/python/src/common/testreport.py

@@ -463,6 +463,8 @@ def generate_result(self, result):
       result_class = 'result-test-result-compliant'
     elif result['result'] == 'Error':
       result_class = 'result-test-result-error'
+    elif result['result'] == 'Feature Not Detected':
+      result_class = 'result-test-result-feature-not-detected'
     else:
       result_class = 'result-test-result-skipped'
 
@@ -1014,6 +1016,12 @@ def generate_css(self):
       left: 7.3in;
     }
 
+    .result-test-result-feature-not-detected {
+      background-color: #dadce0;
+      color: #8d5c00;
+      left: 6.92in;
+    }
+
     .result-test-result-non-compliant {
       background-color: #FCE8E6;
       color: #C5221F;

modules/devices/faux-dev/python/src/dns_check.py

@@ -102,16 +102,16 @@ def _exec_tcpdump(self, tcpdump_filter):
 
     LOGGER.debug('tcpdump command: ' + command)
 
-    process = subprocess.Popen(command,
+    with subprocess.Popen(command,
                                universal_newlines=True,
                                shell=True,
                                stdout=subprocess.PIPE,
-                               stderr=subprocess.PIPE)
-    text = str(process.stdout.read()).rstrip()
+                               stderr=subprocess.PIPE) as process:
+      text = str(process.stdout.read()).rstrip()
 
-    LOGGER.debug('tcpdump response: ' + text)
+      LOGGER.debug('tcpdump response: ' + text)
 
-    if text:
-      return text.split('\n')
+      if text:
+        return text.split('\n')
 
     return []

modules/test/base/python/src/test_module.py

@@ -124,18 +124,20 @@ def run_tests(self):
           test['result'] = 'Compliant' if result else 'Non-Compliant'
           test['description'] = 'No description was provided for this test'
         else:
+          # TODO: This is assuming that result is an array but haven't checked
           # Skipped result
           if result[0] is None:
             test['result'] = 'Skipped'
             if len(result) > 1:
               test['description'] = result[1]
             else:
               test['description'] = 'An error occured whilst running this test'
+
           # Compliant / Non-Compliant result
           elif isinstance(result[0], bool):
             test['result'] = 'Compliant' if result[0] else 'Non-Compliant'
-          # Result may be a string, e.g error
-          elif result[0] == 'Error':
+          # Result may be a string, e.g Error, Feature Not Detected
+          elif isinstance(result[0], str):
             test['result'] = result[0]
           else:
             LOGGER.error(f'Unknown result detected: {result[0]}')

modules/test/base/python/src/util.py

@@ -28,20 +28,20 @@ def run_command(cmd, output=True):
   by any return code from the process other than zero."""
 
   success = False
-  process = subprocess.Popen(shlex.split(cmd),
+  with subprocess.Popen(shlex.split(cmd),
                              stdout=subprocess.PIPE,
-                             stderr=subprocess.PIPE)
-  stdout, stderr = process.communicate()
-  if process.returncode != 0 and output:
-    err_msg = f'{stderr.strip()}. Code: {process.returncode}'
-    LOGGER.error('Command Failed: ' + cmd)
-    LOGGER.error('Error: ' + err_msg)
-  else:
-    success = True
-    LOGGER.debug('Command succeeded: ' + cmd)
-  if output:
-    out = stdout.strip().decode('utf-8')
-    LOGGER.debug('Command output: ' + out)
-    return out, stderr
-  else:
-    return success
+                             stderr=subprocess.PIPE) as process:
+    stdout, stderr = process.communicate()
+    if process.returncode != 0 and output:
+      err_msg = f'{stderr.strip()}. Code: {process.returncode}'
+      LOGGER.error('Command Failed: ' + cmd)
+      LOGGER.error('Error: ' + err_msg)
+    else:
+      success = True
+      LOGGER.debug('Command succeeded: ' + cmd)
+    if output:
+      out = stdout.strip().decode('utf-8')
+      LOGGER.debug('Command output: ' + out)
+      return out, stderr
+    else:
+      return success

modules/test/dns/conf/module_config.json

@@ -27,7 +27,7 @@
         "name": "dns.network.from_dhcp",
         "test_description": "Verify the device allows for a DNS server to be entered automatically", 
         "expected_behavior": "The device sends DNS requests to the DNS server provided by the DHCP server",
-        "required_result": "Roadmap",
+        "required_result": "Informational",
         "recommendations": [
           "Install a DNS client that supports fetching DNS servers from DHCP options"
         ]

modules/test/dns/python/src/dns_module.py

@@ -196,7 +196,6 @@ def _has_dns_traffic(self, tcpdump_filter):
 
   def _dns_network_from_dhcp(self):
     LOGGER.info('Running dns.network.from_dhcp')
-    result = None
     LOGGER.info('Checking DNS traffic for configured DHCP DNS server: ' +
                 self._dns_server)
 
@@ -213,46 +212,43 @@ def _dns_network_from_dhcp(self):
 
     if dns_packets_local or dns_packets_not_local:
       if dns_packets_not_local:
-        result = False, 'DNS traffic detected to non-DHCP provided server'
+        description = 'DNS traffic detected to non-DHCP provided server'
       else:
         LOGGER.info('DNS traffic detected only to configured DHCP DNS server')
-        result = True, 'DNS traffic detected only to DHCP provided server'
+        description = 'DNS traffic detected only to DHCP provided server'
     else:
       LOGGER.info('No DNS traffic detected from the device')
-      result = None, 'No DNS traffic detected from the device'
-    return result
+      description = 'No DNS traffic detected from the device'
+    return 'Informational', description
 
   def _dns_network_hostname_resolution(self):
     LOGGER.info('Running dns.network.hostname_resolution')
-    result = None
     LOGGER.info('Checking DNS traffic from device: ' + self._device_mac)
 
     # Check if the device DNS traffic
     tcpdump_filter = f'dst port 53 and ether src {self._device_mac}'
-    dns_packetes = self._has_dns_traffic(tcpdump_filter=tcpdump_filter)
+    dns_packets = self._has_dns_traffic(tcpdump_filter=tcpdump_filter)
 
-    if dns_packetes:
+    if dns_packets:
       LOGGER.info('DNS traffic detected from device')
       result = True, 'DNS traffic detected from device'
     else:
       LOGGER.info('No DNS traffic detected from the device')
       result = False, 'No DNS traffic detected from the device'
     return result
 
-  ## TODO: This test should always return 'Informational' result
   def _dns_mdns(self):
     LOGGER.info('Running dns.mdns')
-    result = None
     # Check if the device sends any MDNS traffic
     tcpdump_filter = f'udp port 5353 and ether src {self._device_mac}'
-    dns_packetes = self._has_dns_traffic(tcpdump_filter=tcpdump_filter)
+    dns_packets = self._has_dns_traffic(tcpdump_filter=tcpdump_filter)
 
-    if dns_packetes:
+    if dns_packets:
       LOGGER.info('MDNS traffic detected from device')
-      result = True, 'MDNS traffic detected from device'
+      result = 'Informational', 'MDNS traffic detected from device'
     else:
       LOGGER.info('No MDNS traffic detected from the device')
-      result = True, 'No MDNS traffic detected from the device'
+      result = 'Informational', 'No MDNS traffic detected from the device'
     return result
 
   def _exec_tcpdump(self, tcpdump_filter, capture_file):
@@ -267,16 +263,16 @@ def _exec_tcpdump(self, tcpdump_filter, capture_file):
 
     LOGGER.debug('tcpdump command: ' + command)
 
-    process = subprocess.Popen(command,
+    with subprocess.Popen(command,
                                universal_newlines=True,
                                shell=True,
                                stdout=subprocess.PIPE,
-                               stderr=subprocess.PIPE)
-    text = str(process.stdout.read()).rstrip()
+                               stderr=subprocess.PIPE) as process:
+      text = str(process.stdout.read()).rstrip()
 
-    LOGGER.debug('tcpdump response: ' + text)
+      LOGGER.debug('tcpdump response: ' + text)
 
-    if text:
-      return text.split('\n')
+      if text:
+        return text.split('\n')
 
     return []

modules/test/ntp/python/src/ntp_module.py

@@ -208,7 +208,6 @@ def extract_ntp_data(self):
 
   def _ntp_network_ntp_support(self):
     LOGGER.info('Running ntp.network.ntp_support')
-    result = None
     packet_capture = (rdpcap(STARTUP_CAPTURE_FILE) +
                       rdpcap(MONITOR_CAPTURE_FILE) +
                       rdpcap(NTP_SERVER_CAPTURE_FILE))
@@ -245,7 +244,6 @@ def _ntp_network_ntp_support(self):
 
   def _ntp_network_ntp_dhcp(self):
     LOGGER.info('Running ntp.network.ntp_dhcp')
-    result = None
     packet_capture = (rdpcap(STARTUP_CAPTURE_FILE) +
                       rdpcap(MONITOR_CAPTURE_FILE) +
                       rdpcap(NTP_SERVER_CAPTURE_FILE))
@@ -273,11 +271,12 @@ def _ntp_network_ntp_dhcp(self):
         result = False, ('Device sent NTP request to DHCP provided ' +
                          'server and non-DHCP provided server')
       elif ntp_to_remote:
-        result = False, 'Device sent NTP request to non-DHCP provided server'
+        result = ('Feature Not Detected',
+          'Device sent NTP request to non-DHCP provided server')
       elif ntp_to_local:
         result = True, 'Device sent NTP request to DHCP provided server'
     else:
-      result = False, 'Device has not sent any NTP requests'
+      result = 'Feature Not Detected', 'Device has not sent any NTP requests'
 
     LOGGER.info(result[1])
     return result

modules/test/protocol/python/src/protocol_bacnet.py

@@ -15,7 +15,10 @@
 
 import BAC0
 import logging
-from BAC0.core.io.IOExceptions import UnknownPropertyError, ReadPropertyException, NoResponseFromController, DeviceNotConnected
+from BAC0.core.io.IOExceptions import (UnknownPropertyError,
+                                       ReadPropertyException,
+                                       NoResponseFromController,
+                                       DeviceNotConnected)
 
 LOGGER = None
 BAC0_LOG = '/root/.BAC0/BAC0.log'
@@ -54,23 +57,21 @@ def discover(self, local_ip=None):
 
   # Check if the device being tested is in the discovered devices list
   def validate_device(self, local_ip, device_ip):
-    result = None
     LOGGER.info('Validating BACnet device: ' + device_ip)
     self.discover(local_ip + '/24')
-    LOGGER.info('BACnet Devices Found: ' + str(len(self.devices)))
+    LOGGER.info('BACnet devices found: ' + str(len(self.devices)))
     if len(self.devices) > 0:
-      # Load a fail result initially and pass only
-      # if we can validate it's the right device responding
-      result = False, ('Could not confirm discovered BACnet device is the ' +
-                       'same as device being tested')
+      result = (False,
+                'BACnet device was found but was not device under test')
       for device in self.devices:
         address = device[2]
         LOGGER.info('Checking device: ' + str(device))
         if device_ip in address:
-          result = True, 'Device IP matches discovered device'
+          result = True, 'BACnet device discovered'
           break
     else:
-      result = None, 'BACnet discovery could not resolve any devices'
+      result = ('Feature Not Detected',
+        'BACnet device could not be discovered')
     if result is not None:
       LOGGER.info(result[1])
     return result
@@ -85,7 +86,7 @@ def validate_protocol_version(self, device_ip, device_id):
       protocol_version = f'{version}.{revision}'
       result = True
       result_description = (
-          f'BACnet protocol version detected: {protocol_version}')
+          f'Device uses BACnet version {protocol_version}')
     except (UnknownPropertyError, ReadPropertyException,
             NoResponseFromController, DeviceNotConnected) as e:
       result = False

modules/test/protocol/python/src/protocol_modbus.py

@@ -267,7 +267,7 @@ def validate_device(self):
       compliant = (holding_reg is not None or input_reg is not None
                    or coils is not None or discrete_inputs is not None)
     else:
-      compliant = False
+      compliant = None
       details = 'Failed to establish Modbus connection to device'
     result = compliant, details
     return result

modules/test/protocol/python/src/protocol_module.py

@@ -25,6 +25,7 @@ class ProtocolModule(TestModule):
   """Protocol Test module"""
 
   def __init__(self, module):
+    self._supports_bacnet = False
     super().__init__(module_name=module, log_name=LOG_NAME)
     global LOGGER
     LOGGER = self._get_logger()
@@ -48,8 +49,10 @@ def _protocol_valid_bacnet(self):
     if local_address:
       result = self._bacnet.validate_device(local_address,
                                             self._device_ipv4_addr)
+      if result[0]:
+        self._supports_bacnet = True
     else:
-      result = None, 'Could not resolve test container IP for BACnet discovery'
+      result = 'Error', 'Failed to perform BACnet discovery'
     return result
 
   def _protocol_bacnet_version(self):
@@ -59,20 +62,23 @@ def _protocol_bacnet_version(self):
     this test can pass.
     """
     LOGGER.info('Running protocol.bacnet.version')
-    result_status = None
-    result_description = ''
+    result_status = 'Feature Not Detected'
+    result_description = 'Device did not respond to BACnet discovery'
+
+    # Do not run test if device does not support BACnet
+    if not self._supports_bacnet:
+      return result_status, result_description
 
     if len(self._bacnet.devices) > 0:
       for device in self._bacnet.devices:
-        LOGGER.info(f'Checking BACnet version for device: {device}')
         if self._device_ipv4_addr in device[2]:
+          LOGGER.debug(f'Checking BACnet version for device: {device}')
           result_status, result_description = \
-            self._bacnet.validate_protocol_version( device[2], device[3])
+            self._bacnet.validate_protocol_version(device[2], device[3])
           break
         else:
-          LOGGER.info('Device does not match expected IP address, skipping')
-    else:
-      result_description = 'No BACnet devices discovered.'
+          LOGGER.debug('Device does not match expected IP address, skipping')
+
     LOGGER.info(result_description)
     return result_status, result_description
 
@@ -81,15 +87,16 @@ def _protocol_valid_modbus(self, config):
     # Extract basic device connection information
     modbus = Modbus(log=LOGGER, device_ip=self._device_ipv4_addr, config=config)
     results = modbus.validate_device()
+
     # Determine results and return proper messaging and details
-    description = ''
     if results[0] is None:
-      description = 'No modbus connection could be made'
+      result = ('Feature Not Detected',
+                'Device did not respond to Modbus connection')
     elif results[0]:
-      description = 'Valid modbus communication detected'
+      result = True, 'Valid modbus communication detected'
     else:
-      description = 'Failed to confirm valid modbus communication'
-    return results[0], description,  results[1]
+      result = False, 'Failed to confirm valid modbus communication'
+    return result, results[1]
 
   def get_local_ip(self, interface_name):
     try:

modules/test/tls/python/src/run.py

@@ -66,4 +66,4 @@ def run():
 
 
 if __name__ == '__main__':
-  run()
+  run()