diff --git a/conformance_tests.yaml b/conformance_tests.yaml
index 87f063c6..d73d6160 100644
--- a/conformance_tests.yaml
+++ b/conformance_tests.yaml
@@ -3481,3 +3481,11 @@
   output:
     their_name: "Foo Bar"
   tags: [ require, command_line_tool ]
+
+- id: input_directory_with_executable_file
+  job: tests/input_dir_with_executable_file.yaml
+  tool: tests/input_dir_with_executable_file.cwl
+  doc: Executable files in input directories retain their execute permissions
+  output:
+    args: [ "1", "two", "3" ]
+  tags: [ required, command_line_tool ]
\ No newline at end of file
diff --git a/tests/args.py b/tests/args.py
index 1baf6a09..4e29cc99 100755
--- a/tests/args.py
+++ b/tests/args.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import sys
 import json
 import os
diff --git a/tests/cat1-testcli.cwl b/tests/cat1-testcli.cwl
index b82ffbed..d817a95a 100755
--- a/tests/cat1-testcli.cwl
+++ b/tests/cat1-testcli.cwl
@@ -6,7 +6,7 @@
     "hints": [
         {
             "class": "DockerRequirement",
-            "dockerPull": "python:2-slim"
+            "dockerPull": "docker.io/python:3-slim"
         }
     ],
     "inputs": [
diff --git a/tests/import_schema-def_packed.cwl b/tests/import_schema-def_packed.cwl
index 0757ace6..bd5fea13 100644
--- a/tests/import_schema-def_packed.cwl
+++ b/tests/import_schema-def_packed.cwl
@@ -80,7 +80,7 @@
             "id": "#touch.cwl",
             "hints": [
                 {
-                    "dockerPull": "debian:stretch-slim",
+                    "dockerPull": "docker.io/debian:stable-slim",
                     "class": "DockerRequirement"
                 }
             ]
diff --git a/tests/index.py b/tests/index.py
index a54edf93..3f63f025 100755
--- a/tests/index.py
+++ b/tests/index.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # Toy program to generate inverted index of word to line.
 # Takes input text file on stdin and prints output index on stdout.
diff --git a/tests/input_dir_with_executable_file.cwl b/tests/input_dir_with_executable_file.cwl
new file mode 100644
index 00000000..d5c2d59f
--- /dev/null
+++ b/tests/input_dir_with_executable_file.cwl
@@ -0,0 +1,21 @@
+#!/usr/bin/env cwl-runner
+cwlVersion: v1.2
+class: CommandLineTool
+
+hints:
+  DockerRequirement:
+    dockerPull: docker.io/python:3-slim
+
+inputs:
+  dir_with_executable_file: Directory
+
+baseCommand: []
+
+arguments:
+ - $(inputs.dir_with_executable_file.path)/args.py
+ - "1"
+ - two
+ - "3"
+
+outputs:
+  args: string[]
diff --git a/tests/input_dir_with_executable_file.yaml b/tests/input_dir_with_executable_file.yaml
new file mode 100644
index 00000000..0e2fec43
--- /dev/null
+++ b/tests/input_dir_with_executable_file.yaml
@@ -0,0 +1,6 @@
+dir_with_executable_file:
+  class: Directory
+  basename: foo
+  listing:
+    - class: File
+      path: args.py
diff --git a/tests/revsort-packed.cwl b/tests/revsort-packed.cwl
index eedebe20..4b1014ba 100644
--- a/tests/revsort-packed.cwl
+++ b/tests/revsort-packed.cwl
@@ -7,7 +7,7 @@
             "hints": [
                 {
                     "class": "DockerRequirement",
-                    "dockerPull": "debian:stretch-slim"
+                    "dockerPull": "docker.io/debian:stable-slim"
                 }
             ],
             "inputs": [
diff --git a/tests/search.py b/tests/search.py
index d402bc7c..378d6b43 100755
--- a/tests/search.py
+++ b/tests/search.py
@@ -1,10 +1,8 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 # Toy program to search inverted index and print out each line the term
 # appears.
 
-from __future__ import print_function
-
 import sys
 
 mainfile = sys.argv[1]
@@ -23,5 +21,5 @@
         for l in main:
             linenum += 1
             if linenum in n:
-                print(linenum, l.rstrip())
+                print(f"{linenum} {l.rstrip()}")
         break
diff --git a/tests/subdirsecondaries/index.py b/tests/subdirsecondaries/index.py
index 77918b5e..46608d23 100755
--- a/tests/subdirsecondaries/index.py
+++ b/tests/subdirsecondaries/index.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 
 # Toy program to generate inverted index of word to line.
 # Takes input text file on stdin and prints output index on stdout.
diff --git a/tests/tmap-tool.cwl b/tests/tmap-tool.cwl
index 58f75391..35fd986c 100755
--- a/tests/tmap-tool.cwl
+++ b/tests/tmap-tool.cwl
@@ -6,7 +6,7 @@
     "hints": [
         {
             "class": "DockerRequirement",
-            "dockerPull": "python:2-slim"
+            "dockerPull": "docker.io/python:3-slim"
         }
     ],
     "inputs": [