diff --git a/.github/workflows/gh-pages.yaml b/.github/workflows/gh-pages.yaml index 9c0ebcce..fcec7837 100644 --- a/.github/workflows/gh-pages.yaml +++ b/.github/workflows/gh-pages.yaml @@ -17,7 +17,7 @@ jobs: - name: Install apt packages run: | - sudo apt-get install -y graphviz + sudo apt-get install -y graphviz tree - name: Set up Python uses: actions/setup-python@v4 diff --git a/.gitignore b/.gitignore index dc47f1bc..6063055d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ _site .Rhistory .RData _build/ +build/ *.egg-info/ src/_includes/cwl/**/output.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 70b76ed9..06a0000c 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,6 +9,7 @@ build: nodejs: "16" apt_packages: - graphviz + - tree sphinx: configuration: src/conf.py diff --git a/.zenodo.json b/.zenodo.json index 5195b19e..a9aa97e9 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -6,18 +6,135 @@ "publication_type": "softwaredocumentation", "creators": [ { - "affiliation": "European Molecular Biology Laboratory", - "orcid": "https://orcid.org/0000-0003-1766-456X", - "name": "Toby Hodges" + "name": "Alain Domissy" + }, + { + "name": "Andrew Jesaitis" + }, + { + "name": "bamiwoaluko" + }, + { + "orcid": "https://orcid.org/0000-0002-6206-4638>", + "name": "Benjamin Carr" + }, + { + "name": "Michele Mastropietro", + "orcid": "https://orcid.org/0000-0002-6324-5713" + }, + { + "name": "Emannuel Fernandes de Oliveira Carvalho" + }, + { + "name": "Eugene Miloslavsky" + }, + { + "name": "Robin Long", + "orcid": "https://orcid.org/0000-0003-2249-645X" + }, + { + "name": "Ngumih Fien" + }, + { + "name": "gathoni-k" + }, + { + "name": "Henry Liu" + }, + { + "name": "Marijke J. van Baren" }, { - "affiliation": "Common Workflow Language project", + "name": "jessMaia" + }, + { + "name": "Bruno P. Kinoshita", + "orcid": "https://orcid.org/0000-0001-8250-4074" + }, + { + "name": "Kaushik Ghose", + "orcid": "https://orcid.org/0000-0003-2933-1260" + }, + { + "name": "Konstantin Taletskiy", + "orcid": "https://orcid.org/0000-0001-6846-9815" + }, + { + "name": "Kushal Beniwal" + }, + { + "name": "Mark A. Jensen", + "orcid": "https://orcid.org/0000-0001-5215-101X" + }, + { + "name": "Levai Mackenzie Ágbàrà" + }, + { + "name": "Manabu Ishii", + "orcid": "https://orcid.org/0000-0002-5843-4712" + }, + { + "name": "medcelerate" + }, + { + "name": "Michael R. Crusoe", "orcid": "https://orcid.org/0000-0002-2961-9670", - "name": "Michael R. Crusoe" + "affiliation": "Common Workflow Language project" + }, + { + "name": "Ruth B. McCole", + "orcid": "https://orcid.org/0000-0002-1861-7055" + }, + { + "name": "sersorrel" + }, + { + "name": "Adrian Sharma" + }, + { + "name": "Steffen Möller", + "orcid": "https://orcid.org/0000-0002-7187-4683" + }, + { + "name": "Oghenemaro Akpobi" + }, + { + "name": "Stian Soiland-Reyes", + "orcid": "https://orcid.org/0000-0001-9842-9718" + }, + { + "name": "Luka Stojanovic" + }, + { + "name": "Sarah Wait Zaranek", + "orcid": "https://orcid.org/0000-0003-4716-9121" + }, + { + "name": "Peter Amstutz", + "orcid": "https://orcid.org/0000-0003-3566-7705" + }, + { + "name": "Toby Hodges", + "affiliation": "European Molecular Biology Laboratory", + "orcid": "https://orcid.org/0000-0003-1766-456X" + }, + { + "name": "Tomoya Tanjo", + "orcid": "https://orcid.org/0000-0002-4421-9659" + }, + { + "name": "Daiki Tsuchiya", + "orcid": "https://orcid.org/0000-0003-2112-9316" + }, + { + "name": "Wolfgang Gerlach", + "orcid": "https://orcid.org/0000-0002-1480-1242" + }, + { + "name": "Zipho Mashologu" } ], "access_right": "open", - "notes": "Based off of the excellent Software/Data Carpentry templates: https://doi.org/10.5281/zenodo.838778", "related_identifiers": [ { "scheme": "url", diff --git a/AUTHORS.md b/AUTHORS.md index 04e1f5ab..6a2a3a58 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -1 +1,43 @@ -FIXME: list authors' names and email addresses. \ No newline at end of file +Thank you to the following contributors (in alphabetical order by user name): +* Alain Domissy (@alaindomissy) +* Andrew Jesaitis (@andrewjesaitis) +* @bamiwoaluko +* Benjamin Carr (@BenjaminHCCarr) +* Michele Mastropietro (@elehcim) +* Emannuel Fernandes de Oliveira Carvalho (@emannuelOC) +* Eugene Miloslavsky (@emiloslavsky) +* Robin Long (@longr) +* Ngumih Fien (@Fienne) +* @gathoni-k +* Henry Liu (@HenryLiu0) +* Marijke J. van Baren (@Jeltje) +* @jessMaia +* Bruno P. Kinoshita (@kinow) +* Kaushik Ghose (@kghose) +* Konstantin Taletskiy (@ktaletsk) +* Kushal Beniwal (@KushalBeniwal) +* Mark A. Jensen (@majensen) +* Levai Mackenzie Ágbàrà (@Mackenzie-OO7) +* Manabu Ishii (@manabuishii) +* @medcelerate +* Michael Crusoe (@mr-c) +* Ruth B. McCole (@rmccole) +* @sersorrel +* Adrian Sharma (@sharmatime) +* Steffen Möller (@smoe) https://orcid.org/0000-0002-7187-4683 +* Oghenemaro Akpobi (@Smyja) +* Stian Soiland-Reyes (@stain) +* Luka Stojanovic (@StarvingMarvin) +* Sarah Wait Zaranek (@swzCuroverse) https://orcid.org/0000-0003-4716-9121 +* Peter Amstutz (@tetron) +* Toby Hodges (@tobyhodges) +* Tomoya Tanjo (@tom-tan) +* Daiki Tsuchiya (@tsuchi8) +* Wolfgang Gerlach (@wgerlach) +* Zipho Mashologu (@zipho) + +Thanks to: +* [Common Workflow Language community](https://www.commonwl.org/community/) + and the members of the CWL forums and chat rooms. +* [ELIXIR](https://elixir-europe.org/), and ELIXIR Netherlands +* CZI Essential Open Source Software for Science (EOSS) diff --git a/CITATION.md b/CITATION.md index d1ff4cac..985b18af 100644 --- a/CITATION.md +++ b/CITATION.md @@ -1,9 +1,13 @@ # Citation -To cite the Common Workflow Language and the CWL project in a scholary work, please use: +To cite the Common Workflow Language project in a scholary work, please use: -Michael R. Crusoe, Sanne Abeln, Alexandru Iosup, Peter Amstutz, John Chilton, Nebojša Tijanić, Hervé Ménager, Stian Soiland-Reyes, Bogdan Gavrilović, Carole Goble, and The CWL Community. 2022. Methods included: standardizing computational reuse and portability with the Common Workflow Language. Commun. ACM 65, 6 (June 2022), 54–63. https://doi.org/10.1145/3486897 +Michael R. Crusoe, Sanne Abeln, Alexandru Iosup, Peter Amstutz, John Chilton, Nebojša Tijanić, +Hervé Ménager, Stian Soiland-Reyes, Bogdan Gavrilović, Carole Goble, and The CWL Community. 2022. +Methods Included: Standardizing Computational Reuse and Portability with the Common Workflow Language. +Commun. ACM 65, 6 (June 2022), 54–63. https://doi.org/10.1145/3486897 +``` @article{10.1145/3486897, author = {Crusoe, Michael R. and Abeln, Sanne and Iosup, Alexandru and Amstutz, Peter and Chilton, John and Tijani\'{c}, Neboj\v{s}a and M\'{e}nager, Herv\'{e} and Soiland-Reyes, Stian and Gavrilovi\'{c}, Bogdan and Goble, Carole and Community, The CWL}, title = {Methods Included: Standardizing Computational Reuse and Portability with the Common Workflow Language}, @@ -22,6 +26,7 @@ month = {may}, pages = {54–63}, numpages = {10} } +``` To cite just the CWL standards themselves, please use the following inclusive of the DOI: @@ -32,6 +37,7 @@ Stojanovic, Luka (2016): Common Workflow Language, v1.0. Specification, Common Workflow Language working group. https://w3id.org/cwl/v1.0/ https://dx.doi.org/10.6084/m9.figshare.3115156.v2 +``` @data{cwl, doi = {10.6084/m9.figshare.3115156.v2}, url = {https://doi.org/10.6084/m9.figshare.3115156.v2}, @@ -45,8 +51,9 @@ Luka Stojanovic title = {Common Workflow Language, v1.0}, year = {2016} } +``` -# the above is from https://github.com/common-workflow-language/common-workflow-language/blob/main/CITATION +[//]: # (the above is from https://github.com/common-workflow-language/common-workflow-language/blob/main/CITATION.cff) Our preferred DOI for the "CWL User Guide" is http://doi.org/10.5281/zenodo.840129 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f6a9349..567f0da3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,9 +41,14 @@ and to meet some of our community members. 3. If you are comfortable with Git, and would like to add or change material, you can submit a pull request (PR). - Instructions for doing this are [included below][#using-github]. + Instructions for doing this are [included below][using-github]. + +4. To build and run the user guide locally, see **Building** below + +**Note:** The published version of the user guide is built from the `release` branch. +New changes are gathered on the default (`main`) branch which is built at +as a preview. Once we collect many changes from the `main` branch, we merge them into the `release` branch. -4. To build and run the user guide locally, see [building][#building]. Pull requests include an automatic preview provided by [ReadTheDocs](https://readthedocs.org/projects/common-workflow-languageuser-guide/). @@ -101,11 +106,13 @@ in `Makefile`). # Create and activate a virtual environment python -m venv venv source venv/bin/activate +# update the version of pip, setuptools, and wheel +(venv) pip install -U pip setuptools wheel # Install the dependencies in your virtual environment -(venv) pip install -e .[all] +(venv) pip install .[all] # Create the HTML to visualize locally (venv) make html -(venv) firefox _build/index.html +(venv) open _build/index.html # Or you can start a serve that watches for local file changes (venv) make watch # Open in your browser @@ -118,6 +125,56 @@ source venv/bin/activate > correctly, but failure to deploy the production version after the pull request > gets merged. +## Style Guide + +We must use the phrase "CWL standards" or "CWL open standards" when talking about CWL. +We must use the word "specification" only when talking specifically about the CWL +specification document. + +Whenever a page is updated we must verify that it does not break existing +links, both internal and external. The `make html` command will fail if Sphinx detects broken links. +It only works for links managed by Sphinx (i.e. table of contents links, +or links to Markdown pages). For simple HTML links (e.g. `< href=>` or +markdown external links) pull request reviewers must verify that links +are still working after the change. + +Use “tool description” not “tool wrapper” for describing the first argument +given to the `cwl-runner` or `cwltool` commands. + +### Code examples + +To include code into a Markdown file you have two options. For external files use +the following command: + +```` +```{literalinclude} /_includes/cwl/hello_world.cwl +:language: cwl +``` +```` + +For code examples in the same page, you can use fence blocks. + +```` +```bash +echo "Hello world" +``` +```` + +If you would like to customize the syntax highlighting styles +you will have to customize the Sphinx and Pygments settings. +To preview Pygments output with different styles, use their +[Pygments demo tool](https://pygments.org/demo/). + +### Creating Links + +Sphinx and the theme are configured to auto-generate anchor slug +links for sections. So sections like ``## cwl standard`` are translated +into an anchor link `#cwl-standard`. + +If you are having trouble with links to sections or code blocks, it might +be due to duplicated sections, or to spaces or other characters. To +preview the generated links, use the `myst-anchors` tool. + ## Other Resources General discussion of [Common Workflow Language][cwl-site] project @@ -128,7 +185,9 @@ which everyone is welcome to join. [github]: https://github.com [github-flow]: https://guides.github.com/introduction/flow/ [github-join]: https://github.com/join -[how-contribute]: https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github +[how-contribute]: https://docs.github.com/en/get-started/quickstart/contributing-to-projects [issues]: https://github.com/common-workflow-language/user_guide/issues +[cwl-issues]: https://github.com/common-workflow-language/common-workflow-language/issues [repo]: https://github.com/common-workflow-language/user_guide [cwl-site]: https://www.commonwl.org/ +[using-github]: https://docs.github.com/en/get-started/ diff --git a/README.md b/README.md index 87771ef5..f40921c5 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,14 @@ -[![DOI](https://zenodo.org/badge/89621457.svg)](https://zenodo.org/badge/latestdoi/89621457) +[![DOI for the latest version](https://zenodo.org/badge/89621457.svg)](https://zenodo.org/badge/latestdoi/89621457) [![Syntax Check](https://travis-ci.org/common-workflow-language/user_guide.svg?branch=main)](https://travis-ci.org/common-workflow-language/user_guide) -[User guide for CWL v1.2.0](https://www.commonwl.org/user_guide/) + +This is the source of the official user guide for the Common Workflow Language standards. + +Releases are published at from the `release` branch. + +Preview of the next release is published at +based upon the default branch (`main`) using the Read The Docs (RTD) service. Original source: https://github.com/common-workflow-language/common-workflow-language/blob/a2a8a08b8c8d56f8f2ca6284ca4e9cbf23d19346/v1.0/UserGuide.yml @@ -15,98 +21,6 @@ To edit the user guide: We'd like to ask you to familiarize yourself with our [Contribution Guide](CONTRIBUTING.md). -## Style Guide - -We must use CWL standards or CWL open standards when talking about CWL. -We must use specification only when talking specifically about the CWL -specification document. - -Whenever a page is updated we must verify that it does not break existing -links. The `make html` command will fail if Sphinx detects broken links. -It only works for links managed by Sphinx (i.e. table of contents links, -or links to Markdown pages). For simple HTML links (e.g. `< href=>` or -markdown external links) pull request reviewers must verify that links -are still working after the change. - -Use “tool description” not “tool wrapper” for describing the first argument -given to the `cwl-runner` or `cwltool` commands. - -### Code examples - -To include code into a Markdown file you have two options. For external files use -the following command: - -```` -```{literalinclude} /_includes/cwl/hello_world.cwl -:language: cwl -``` -```` - -For code examples in the same page, you can use fence blocks. - -```` -```bash -echo "Hello world" -``` -```` - -If you would like to customize the syntax highlighting styles -you will have to customize the Sphinx and Pygments settings. -To preview Pygments output with different styles, use their -[Pygments demo tool](https://pygments.org/demo/). - -### Creating Links - -Sphinx and the theme are configured to auto-generate anchor slug -links for sections. So sections like ``## cwl standard`` are translated -into an anchor link `#cwl-standard`. - -If you are having trouble with links to sections or code blocks, it might -be due to duplicated sections, or to spaces or other characters. To -preview the generated links, use the `myst-anchors` tool. - -```bash -$ (venv) myst-anchors basic-concepts.md -

-

-

-

-``` - -You can also create reference anchor links anywhere on the page with -``(test)=``, which can be used in the page as `#test` (these do not appear -in the `myst-anchor` output). - -> Links to anchors in pages work only for Sphinx's autogenerated anchor links. -> If you have an anchor for, for example, a code-block, that might cause the -> build to fail: https://github.com/executablebooks/MyST-Parser/issues/564 - -## Extensions - -We use MyST Parser with Sphinx. This gives us the best of both Sphinx and Markdown, -while also supporting reStructuredText, Sphinx, and MyST extensions. - -### String Substitutions - -For convenience, we have the currently supported version of the specification as a -constant in `conf.py`. We have it in two forms: - -- Markdown preformatted, i.e. \`v0.0\` which is formatted as `v0.0` -- Plain text, i.e. v0.0 - -Note that String Substitutions do not work with links. As workaround, you can use -string formatting or replacements. - -``` -The CWL {{ cwl_version }} Specification: {{ ''.format(cwl_version_text) }} -``` - -For more: - -- -- -- - ## Authors A list of contributors to these materials can be found in [AUTHORS](AUTHORS.md) diff --git a/cwl/sphinx/runcmd.py b/cwl/sphinx/runcmd.py index e9cb3016..ff8bb245 100644 --- a/cwl/sphinx/runcmd.py +++ b/cwl/sphinx/runcmd.py @@ -169,9 +169,20 @@ def run(self): r = match.group("replacement").replace("\\", "") output = re.sub(p, r, output) + # Note: Sphinx's CodeBlock directive expects an array of command-line + # output lines: https://github.com/sphinx-doc/sphinx/blob/c51a88da8b7b40e8d8cbdb1fce85ca2346b2b59a/sphinx/directives/code.py#L114 + # But the runcmd original code was simply wrapping a string + # containing \n in the text as a one-element array, e.g. + # ["cwltool --debug ...\ncwltool Version..."]. + # That caused the output to be correctly rendered, but the + # emphasize-lines directive parameter to fail if the lines were + # anything greater than 0 (as the self.content array had 1 elem). + # See: https://github.com/common-workflow-language/user_guide/issues/269 + output = output.split("\n") + # Set up our arguments to run the CodeBlock parent run function self.arguments[0] = syntax - self.content = [output] + self.content = output node = super(RunCmdDirective, self).run() return node diff --git a/setup.cfg b/setup.cfg index 0d0a6f7f..f4315385 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,7 +32,8 @@ packages = find_namespace: include_package_data = True python_requires = >=3.6 install_requires = - cwl_runner + cwltool + cwlref-runner cwl-utils==0.* myst-parser==0.* pydata-sphinx-theme==0.* diff --git a/src/_includes/cwl/expressions/custom-functions.js b/src/_includes/cwl/expressions/custom-functions.js new file mode 100644 index 00000000..6e9d05f7 --- /dev/null +++ b/src/_includes/cwl/expressions/custom-functions.js @@ -0,0 +1,18 @@ +/** + * Capitalize each word passed. Will split the text by spaces. + * For instance, given "hello world", it returns "Hello World". + * + * @param {String} message - The input message. + * @return {String} the message with each word with its initial letter capitalized. + */ +function capitalizeWords (message) { + if (message === undefined || message === null || typeof message !== 'string' || message.trim().length === 0) { + return ''; + } + return message + .split(' ') + .map(function (token) { + return token.charAt(0).toUpperCase() + token.slice(1); + }) + .join(' '); +} diff --git a/src/_includes/cwl/expressions/hello-world-expressionlib-external.cwl b/src/_includes/cwl/expressions/hello-world-expressionlib-external.cwl new file mode 100644 index 00000000..9cc416af --- /dev/null +++ b/src/_includes/cwl/expressions/hello-world-expressionlib-external.cwl @@ -0,0 +1,16 @@ +cwlVersion: v1.2 +class: CommandLineTool +requirements: + - class: InlineJavascriptRequirement + expressionLib: + - { $include: custom-functions.js } + +baseCommand: echo + +inputs: + message: + type: string + +arguments: [$( capitalizeWords(inputs.message) )] + +outputs: [] diff --git a/src/_includes/cwl/expressions/hello-world-expressionlib-inline.cwl b/src/_includes/cwl/expressions/hello-world-expressionlib-inline.cwl new file mode 100644 index 00000000..113b65f7 --- /dev/null +++ b/src/_includes/cwl/expressions/hello-world-expressionlib-inline.cwl @@ -0,0 +1,34 @@ +cwlVersion: v1.2 +class: CommandLineTool +requirements: + - class: InlineJavascriptRequirement + expressionLib: + - | + /** + * Capitalize each word passed. Will split the text by spaces. + * For instance, given "hello world", it returns "Hello World". + * + * @param {String} message - The input message. + * @return {String} the message with each word with its initial letter capitalized. + */ + function capitalizeWords (message) { + if (message === undefined || message === null || typeof message !== 'string' || message.trim().length === 0) { + return ''; + } + return message + .split(' ') + .map(function (token) { + return token.charAt(0).toUpperCase() + token.slice(1); + }) + .join(' '); + } + +baseCommand: echo + +inputs: + message: + type: string + +arguments: [$( capitalizeWords(inputs.message) )] + +outputs: [] diff --git a/src/_includes/cwl/expressions/hello-world-expressionlib.cwl b/src/_includes/cwl/expressions/hello-world-expressionlib.cwl new file mode 100644 index 00000000..21ea73a2 --- /dev/null +++ b/src/_includes/cwl/expressions/hello-world-expressionlib.cwl @@ -0,0 +1,27 @@ +cwlVersion: v1.2 +class: CommandLineTool +requirements: + - class: InlineJavascriptRequirement + expressionLib: + - { $include: custom-functions.js } + - | + /** + * A merely illustrative example function that uses a function + * from the included custom-functions.js file to create a + * Hello World message. + * + * @param {Object} message - CWL document input message + */ + var createHelloWorldMessage = function (message) { + return capitalizeWords(message); + }; + +baseCommand: echo + +inputs: + message: + type: string + +arguments: [$( createHelloWorldMessage(inputs.message) )] + +outputs: [] diff --git a/src/_includes/cwl/inputs/exclusive-parameter-expressions.cwl b/src/_includes/cwl/inputs/exclusive-parameter-expressions.cwl new file mode 100644 index 00000000..dd98d0a5 --- /dev/null +++ b/src/_includes/cwl/inputs/exclusive-parameter-expressions.cwl @@ -0,0 +1,25 @@ +cwlVersion: v1.2 +class: CommandLineTool + +inputs: + file_format: + type: + - 'null' + - name: format_choices + type: enum + symbols: + - auto + - fasta + - fastq + - fasta.gz + - fastq.gz + inputBinding: + position: 0 + prefix: '--format' +outputs: + text_output: + type: string + outputBinding: + outputEval: $(inputs.file_format) + +baseCommand: 'true' diff --git a/src/_includes/cwl/troubleshooting/troubleshooting-wf1-stepb-fixed.cwl b/src/_includes/cwl/troubleshooting/troubleshooting-wf1-stepb-fixed.cwl new file mode 100644 index 00000000..90b7cb94 --- /dev/null +++ b/src/_includes/cwl/troubleshooting/troubleshooting-wf1-stepb-fixed.cwl @@ -0,0 +1,48 @@ +cwlVersion: v1.2 +class: Workflow + +inputs: + text: + type: string + default: 'Hello World' +outputs: + reversed_message: + type: string + outputSource: step_b/reversed_message + +steps: + step_a: + run: + class: CommandLineTool + stdout: stdout.txt + inputs: + text: string + outputs: + step_a_stdout: + type: File + outputBinding: + glob: 'stdout.txt' + baseCommand: echo + arguments: [ '-n', '$(inputs.text)' ] + in: + text: text + out: [step_a_stdout] + step_b: + run: + class: CommandLineTool + stdout: stdout.txt + inputs: + step_a_stdout: File + outputs: + reversed_message: + type: string + outputBinding: + glob: stdout.txt + loadContents: true + outputEval: $(self[0].contents) + baseCommand: rev + arguments: [ $(inputs.step_a_stdout) ] + in: + step_a_stdout: + source: step_a/step_a_stdout + out: [reversed_message] diff --git a/src/_includes/cwl/troubleshooting/troubleshooting-wf1.cwl b/src/_includes/cwl/troubleshooting/troubleshooting-wf1.cwl new file mode 100644 index 00000000..f2ca2a0e --- /dev/null +++ b/src/_includes/cwl/troubleshooting/troubleshooting-wf1.cwl @@ -0,0 +1,47 @@ +cwlVersion: v1.2 +class: Workflow + +inputs: + text: + type: string + default: 'Hello World' +outputs: + reversed_message: + type: string + outputSource: step_b/reversed_message + +steps: + step_a: + run: + class: CommandLineTool + stdout: stdout.txt + inputs: + text: string + outputs: + step_a_stdout: + type: File + outputBinding: + glob: 'stdout.txt' + arguments: ['echo', '-n', '$(inputs.text)'] + in: + text: text + out: [step_a_stdout] + step_b: + run: + class: CommandLineTool + stdout: stdout.txt + inputs: + step_a_stdout: File + outputs: + reversed_message: + type: string + outputBinding: + glob: stdout.txt + loadContents: true + outputEval: $(self[0].contents) + baseCommand: revv + arguments: [ $(inputs.step_a_stdout) ] + in: + step_a_stdout: + source: step_a/step_a_stdout + out: [reversed_message] diff --git a/src/_templates/layout.html b/src/_templates/layout.html index 70133ecd..e55cff18 100644 --- a/src/_templates/layout.html +++ b/src/_templates/layout.html @@ -15,7 +15,7 @@ Source
  • - Cite + Cite
    • diff --git a/src/faq.md b/src/faq.md index 86a74f7f..8e7f41a2 100644 --- a/src/faq.md +++ b/src/faq.md @@ -58,7 +58,7 @@ requirements: ## Rename an output file -This example shows how you can change the name an output file +This example shows how you can change the name of an output file from the default name given to it by a tool: ```yaml @@ -191,7 +191,7 @@ inputs: ## Enum Inputs ⚜️ -For commandline flags that require a specific input as the argument an enum type can be declared in CWL. **Specifying null here is known as long form style. It does the same thing as the question mark on the other inputs.** +For command-line flags that require a specific input as the argument, an enum type can be declared in CWL. **Specifying null here is known as long form style. It does the same thing as the question mark on the other inputs.** ```yaml Format: @@ -211,7 +211,7 @@ Format: ## Record Inputs 📀 -For commandline flags that are either **mutually exclusive** or **dependent** a special record type can be defined. You can also specify null here to create optional inputs. +For command-line flags that are either **mutually exclusive** or **dependent**, a special record type can be defined. You can also specify null here to create optional inputs. ```yaml #Using record inputs to create mutually exclusive inputs @@ -290,7 +290,7 @@ These can be set by using the default field input: default:true ``` -## Concating Strings in Inputs +## Concatenating Strings in Inputs The valueFrom field must be used instead of default. @@ -304,7 +304,7 @@ input: `cwltool` does not allow some characters in filenames by default. -For example, the filename is `a space is here.txt` includes 3 space characters. +For example, the filename `a space is here.txt` includes 3 space characters. ```console ERROR Workflow error, try again with --debug for more information: @@ -364,7 +364,7 @@ inputs: ``` -Fix this error is change `-` (hyphen) to `_` (underscore) +To fix this error, change `-` (hyphen) to `_` (underscore) ```cwl valueFrom: $(inputs.sample_input) @@ -377,7 +377,7 @@ inputs: # ^ changed here ``` -If is not possible to change the input identifier, then you can use an alternative CWL Parameter Reference syntax: +If it is not possible to change the input identifier, then you can use an alternative CWL Parameter Reference syntax: ```cwl valueFrom: $(inputs["sample-input"]) diff --git a/src/index.md b/src/index.md index 310df3af..8e10fa54 100644 --- a/src/index.md +++ b/src/index.md @@ -17,7 +17,7 @@ each page. If you are a beginner user get started with the [Introduction](/introduction/index.md) section. For advanced users the subsections of the -[Core Concepts](/topics/index.md) have detailed information about the +[Topics](/topics/index.md) have detailed information about the most common topics for CWL. The Table of Contents is displayed at the top menu and also on the left sidebar. @@ -28,7 +28,7 @@ the left sidebar. ## Table of Contents ```{toctree} -:maxdepth: 2 +:maxdepth: 3 :numbered: introduction/index.md diff --git a/src/introduction/index.md b/src/introduction/index.md index f2f3a9ad..78d32ad3 100644 --- a/src/introduction/index.md +++ b/src/introduction/index.md @@ -1,8 +1,8 @@ # Introduction -This section will guide you over a short introduction to CWL, +This section will guide you through a short introduction to CWL, the prerequisites for following this user guide, and some -basic concepts that are useful before reading the rest of the +basic concepts that are useful to know before reading the rest of the user guide. ```{toctree} diff --git a/src/introduction/prerequisites.md b/src/introduction/prerequisites.md index 574611fb..f739add6 100644 --- a/src/introduction/prerequisites.md +++ b/src/introduction/prerequisites.md @@ -4,14 +4,15 @@ % writing this documentation. The software and configurations listed in this section are prerequisites for -following this user guide. The CWL Specification is implemented by multiple -CWL Runners. This list of requirements focuses on the `cwltool` runner. You -can use another CWL Runner but the examples may produce a different output. +following this user guide. The CWL standards are implemented by many different +workflow runners and platforms. This list of requirements focuses on the CWL reference runner, +`cwltool`. You can use another CWL compatible runner or workflow systems but the results and +interface may look different (though the exact workflow outputs should be identical). ```{admonition} CWL Implementations -There are many CWL Implementations. Some are complete CWL Runners, -others are plug-ins or extensions to Workflow Engines. We have a better +There are many implementations of the CWL standards. Some are complete CWL runners, +others are plug-ins or extensions to workflow engines. We have a better explanation in the [Implementations](basic-concepts.md#implementations) section. ``` @@ -22,12 +23,14 @@ of the following options for your operating system: - Linux - macOS +- Windows -You can try to use Window Subsystem for Linux 2 (WSL) to follow the -User Guide documentation, but some examples may not work as expected. - -Your operating system also needs Internet access and a recent version -of Python 3. +```{note} +If you are using Windows, you will have to install the Windows Subsystem for Linux 2. +Visit the `cwltool` [documentation](https://github.com/common-workflow-language/cwltool/blob/main/README.rst#ms-windows-users) +for details on installing WSL2. +Your operating system also needs internet access and a recent version of Python (3.6+). +``` ## CWL runner @@ -51,36 +54,35 @@ environment: $ python -m venv venv $ source venv/bin/activate +$ (venv) pip install -U pip setuptools wheel $ (venv) pip install cwltool ``` ```{note} +You can find the `cwl-runner` source code [here](https://github.com/common-workflow-language/cwltool/tree/main/cwlref-runner). Visit the `cwltool` [documentation](https://github.com/common-workflow-language/cwltool#install) for other ways to install `cwltool` with `apt` and `conda`. ``` - -Let's use a simple workflow `true.cwl` with `cwltool`. +Let's use a simple CWl tool description `true.cwl` with `cwltool`. ```{literalinclude} /_includes/cwl/true.cwl -:language: cwl +:language: yaml :caption: "`true.cwl`" :name: true.cwl ``` -The `cwltool` command has an option to validate CWL workflows. It will parse the -CWL workflow, look for syntax errors, and verify that the workflow is compliant -with the CWL specification, without running the workflow. To use it you just need -to pass `--validate` to the `cwltool` command: - -% TODO: maybe figure out a way to avoid /home/kinow/ etc. in the documentation -% to avoid multiple user-names/directory-locations varying in the docs. +The `cwltool` command has an option to validate CWL tool and workflow descriptionss. It will parse the +CWL document, look for syntax errors, and verify that the descriptions are compliant +with the CWL standards, without running it. To validate CWL workflows (or even a +standalone command line tool description like above) pass the `--validate` option +to the `cwltool` command: ```{runcmd} cwltool --validate true.cwl :name: validating-truecwl-with-cwltool :caption: Validating `true.cwl` with `cwltool`. ``` -You can run the CWL workflow now that you know it is valid: +You can run the CWL tool description by omitting the `--validate` option: ```{runcmd} cwltool true.cwl :name: running-true.cwl-with-cwltool @@ -89,26 +91,21 @@ You can run the CWL workflow now that you know it is valid: ### cwl-runner Python module -`cwl-runner` is an implementation agnostic alias for CWL Runners. +`cwl-runner` is an implementation-agnostic alias for CWL Runners. Users can invoke `cwl-runner` instead of invoking a CWL runner like `cwltool` directly. The `cwl-runner` alias command then chooses the correct CWL runner. This is convenient for environments with multiple CWL runners. -The CWL community publishes a Python module with the same name, -`cwl-runner`, that defaults to `cwltool`. `cwl-runner` will be used in -the rest of this user guide. You can use `pip` to install the `cwl-runner` -Python module: +The CWL community publishes a Python package with the name `cwlref-runner` that installs +an alias for `cwltool` under the name `cwl-runner` ```{code-block} console :name: installing-cwlrunner-with-pip -:caption: Installing `cwl-runner` with `pip`. +:caption: Installing `cwl-runner` alias for cwltool with `pip`. -$ pip install cwl-runner +$ pip install cwlref-runner ``` -% TODO: Maybe tell users where the cwl-runner source is? I couldn't find in PYPI as -% it points to the CWL project: https://github.com/common-workflow-language/cwltool/tree/main/cwlref-runner - Now you can validate and run your workflow with `cwl-runner` executable, which will invoke `cwltool`. You should have the same results and output as in the previous section. @@ -174,13 +171,13 @@ auto-complete: - cwl-plugin for IntelliJ - The CWL community also maintains a list of editors and viewers: - + ## Docker % https://github.com/common-workflow-language/user_guide/issues/119 -`cwltool` uses Docker to run workflows or workflow steps with containers. +`cwltool` uses Docker to run tools, workflows, and workflow steps that specify a software container. Follow the instructions in the Docker documentation to install it for your operating system: . diff --git a/src/introduction/quick-start.md b/src/introduction/quick-start.md index 60a6d965..98476fcf 100644 --- a/src/introduction/quick-start.md +++ b/src/introduction/quick-start.md @@ -1,6 +1,6 @@ # Quick Start -This section will show you a brief overview of what is CWL and where you +This section will show you a brief overview of what CWL is, and where you can learn more about it. No previous knowledge of CWL is required, but you must be comfortable following instructions for the command-line. @@ -20,20 +20,20 @@ with comments: ``` The example above is just a wrapper for the `echo` command-line tool. -Running the workflow above with the default input values, produces the +Running the workflow above with the default input values will produce the same result as the command-line `echo "Hello World"`. ```{note} -There is a distinction in CWL between a command-line tool and a workflow. But -for the sake of simplicity we are using the term “workflow” here. You will learn +In CWL, there is a distinction between a command-line tool and a workflow. But +for the sake of simplicity, we are using the term “workflow” here. You will learn more about this in the [basic concepts](basic-concepts.md) section. ``` ## Installing a CWL runner `cwltool` is an implementation of the CWL specification. It is also the -CWL *Reference Runner* for the specification, and compliant with the -latest version of the specification, {{ cwl_version }}. You can install +CWL *Reference Runner* for the specification, and it is compliant with the +latest version of the specification: {{ cwl_version }}. You can install `cwltool` using `pip`: ```{code-block} console @@ -44,9 +44,8 @@ $ pip install cwltool ``` ```{note} -The [prerequisites](prerequisites.md) section contains a more detailed list +If installing the cwltool using the pip command doesn't work for you, the [prerequisites](prerequisites.md) section contains other ways to install `cwltool` and a more detailed list of software and libraries used for following the rest of this user guide. -It also contains other ways to install `cwltool`. ``` ## Running "Hello World" @@ -69,8 +68,8 @@ to how you would change the argument of the `echo` base command: ``` Another way of passing values to your workflow input parameters is via an -*Inputs Object*. This is a file containing the input fields with the -corresponding values. This file can be written in JSON or YAML. For example: +*Inputs Object*. This is a file containing the input fields with their +corresponding values. The Inputs Objects file can be written in JSON or YAML. For example: ```{literalinclude} /_includes/cwl/hello_world-job.json :language: json @@ -86,21 +85,21 @@ You can use this Inputs Object file now to execute the “Hello World” workflo ``` ```{note} -We used a similar file name for the workflow and for the inputs object files. +We used a similar file name for the workflow and for the Inputs Object files. The *-job.json* suffix is very common in Inputs Object files, but it is not -a requirement. You can choose any name for your workflows and inputs object +a requirement. You can choose any name for your workflows and Inputs Object files. ``` ## Learn more -- Continue reading the next sections of this User Guide! -- List of CWL Implementations: -- The `common-workflow-language` organization at GitHub: -- Common Workflow Language at Wikipedia: -- YAML.org: and YAML at Wikipedia: -- The CWL {{ cwl_version }} Specification: {{ ''.format(cwl_version_text) }} -- Workflow management system at Wikipedia: +Continue reading the next sections of this User Guide! +- [List of CWL Implementations](https://www.commonwl.org/implementations). +- The [`common-workflow-language` organization](https://github.com/common-workflow-language) at GitHub. +- [Common Workflow Language at Wikipedia](https://en.wikipedia.org/wiki/Common_Workflow_Language). +- [YAML.org](http://yaml.org/) and [YAML at Wikipedia](https://en.wikipedia.org/wiki/YAML). +- The {{'[CWL Specification VERSION](https://www.commonwl.org/VERSION)'.replace('VERSION', cwl_version_text) }}. +- [Workflow management system at Wikipedia](https://en.wikipedia.org/wiki/Workflow_management_system). % N.B.: Wondering what's up with this syntax in the CWL Specification link above? % It's necessary as MyST Parser does not allow substitutions in links, for more: diff --git a/src/topics/best-practices.md b/src/topics/best-practices.md index 8f134205..fe0d663f 100644 --- a/src/topics/best-practices.md +++ b/src/topics/best-practices.md @@ -8,14 +8,23 @@ all are required. - No `type: string` parameters for names of input or reference files/directories; use `type: File` or `type: Directory` as appropriate. -- Include a license that allows for re-use by anyone, e.g. - [Apache 2.0][apache-license]. If possible, the license should be specified with - its corresponding [SPDX identifier][spdx]. Construct the metadata field for the - licence by providing a URL of the form `https://spdx.org/licenses/[SPDX-ID]` - where `SPDX-ID` is the taken from the list of identifiers linked above. See the - example snippet below for guidance. For non-standard licenses without an SPDX +- A CWL document (in conjunction with any external components like `Dockerfile`s) is software code. + Workflow developers should be aware that the usual rules of software licensing apply to this + document. For example if the workflow is shared publicly, licensing terms have to be clear so that + a future user can understand under what conditions they can run the workflow, modify it and/or + combine it with other workflows. For this reason please consider including a license field in the + document. The authors of this guide urge you to choose a pre-existing license rather than trying + to write your own (see the link below to learn more about choosing a license), and our recommended + practice is to choose a license that allows for re-use by anyone, e.g. [Apache 2.0][apache-license]. + + If possible, the license should be specified with its corresponding [SPDX identifier][spdx]. + Construct the metadata field for the licence by providing a URL of the form + `https://spdx.org/licenses/[SPDX-ID]` where `SPDX-ID` is taken from the list of identifiers + linked above. See the example snippet below for guidance. For non-standard licenses without an SPDX identifier, provide a URL to the license. + Useful reading: "[A Quick Guide to Software Licensing for the Scientist-Programmer][sci-license]" + _Example of metadata field for license with SPDX identifier:_ ```cwl @@ -25,8 +34,8 @@ all are required. # other s: declarations ``` - For more examples of providing metadata within CWL descriptions, see the - Metadata and Authorship section of this User Guide. + For more examples of providing metadata within CWL descriptions, see + [the Metadata and Authorship section of this User Guide](../topics/metadata-and-authorship.md). - Include [attribution information][license-example] for the author(s) of the CWL tool or workflow description. Use unambiguous identifiers like @@ -97,6 +106,7 @@ all are required. [orcid]: https://orcid.org [subworkflow]: https://www.commonwl.org/v1.0/Workflow.html#SubworkflowFeatureRequirement [spdx]: https://spdx.org/licenses/ +[sci-license]: https://doi.org/10.1371/journal.pcbi.1002598 % TODO % diff --git a/src/topics/creating-files-at-runtime.md b/src/topics/creating-files-at-runtime.md index 450c34dc..9a652b62 100644 --- a/src/topics/creating-files-at-runtime.md +++ b/src/topics/creating-files-at-runtime.md @@ -1,10 +1,10 @@ # Creating Files at Runtime Sometimes you need to create a file on the fly from input parameters, -such as tools which expect to read their input configuration from a file +such as tools that expect to read their input configuration from a file rather than the command line parameters, or need a small wrapper shell script. -To generate such files we can use the `InitialWorkDirRequirement`. +To generate such files, we can use the `InitialWorkDirRequirement`. ```{literalinclude} /_includes/cwl/creating-files-at-runtime/createfile.cwl :language: cwl @@ -13,18 +13,18 @@ To generate such files we can use the `InitialWorkDirRequirement`. ``` Any [expressions](../topics/expressions.md) like `$(inputs.message)` are -expanded by the CWL engine before creating the file; -here inserting the value at the input `message`. +expanded by the CWL engine before creating the file. +Here, insert the value at the input `message`. ```{tip} The _CWL expressions_ are independent of any _shell variables_ used later during command line tool invocation. That means that any genuine -need for the character `$` must be **escaped** with `\`, -for instance `\${PREFIX}` above is expanded to `${PREFIX}` in the generated file +need for the character `$` must be **escaped** with `\`. +For instance, `\${PREFIX}` above is expanded to `${PREFIX}` in the generated file to be evaluated by the shell script instead of the CWL engine. ``` -To test the above CWL tool use this job to provide the input value `message`: +To test the above CWL tool, use this job to provide the input value `message`: ```{literalinclude} /_includes/cwl/creating-files-at-runtime/echo-job.yml :language: yaml @@ -32,13 +32,13 @@ To test the above CWL tool use this job to provide the input value `message`: :name: echo-job.yml ``` -Before we run this, lets look at each step in a little more detail. +Before we run this, let us look at each step in a little more detail. The base command `baseCommand: ["sh", "example.sh"]` will execute the command `sh example.sh`. This will run the file we create in the shell. `InitialWorkDirRequirement` requires a `listing`. -As the `listing` is a YAML array we need a `-` on the first line of +As the `listing` is a YAML array, we need a `-` on the first line of each element of the array, in this case we have just one element. `entryname:` can have any value, but it must match what was specified in the `baseCommand`. diff --git a/src/topics/custom-types.md b/src/topics/custom-types.md index dd20d79a..f0b59caf 100644 --- a/src/topics/custom-types.md +++ b/src/topics/custom-types.md @@ -21,10 +21,10 @@ converting a standard biom table file to hd5 format. :name: custom-types.yml ``` -___Note:___ To follow the example below, you need to download the example input file, *rich_sparse_otu_table.biom*. The file is available from [https://raw.githubusercontent.com/common-workflow-language/user_guide/main/_includes/cwl/custom-types/rich_sparse_otu_table.biom](https://raw.githubusercontent.com/common-workflow-language/user_guide/main/_includes/cwl/custom-types/rich_sparse_otu_table.biom) and can be downloaded e.g. via `wget`: +___Note:___ To follow the example below, you need to [download the example input file](https://raw.githubusercontent.com/common-workflow-language/user_guide/main/_includes/cwl/custom-types/rich_sparse_otu_table.biom), *rich_sparse_otu_table.biom* e.g. via `wget`: ```{code-block} console -$ wget https://raw.githubusercontent.com/common-workflow-language/user_guide/main/src/_includes/cwl/custom-types/rich_sparse_otu_table.biom +$ wget https://github.com/common-workflow-language/user_guide/raw/main/src/_includes/cwl/custom-types/rich_sparse_otu_table.biom ``` On line 29, in `inputs:table_type`, a list of allowable table options to be used in the diff --git a/src/topics/expressions.md b/src/topics/expressions.md index 31b3ee9e..eebc027b 100644 --- a/src/topics/expressions.md +++ b/src/topics/expressions.md @@ -11,6 +11,7 @@ When manipulating file names, extensions, paths etc, consider whether one of the [built in `File` properties][file-prop] like `basename`, `nameroot`, `nameext`, etc, could be used instead. See the [list of best practices](best-practices.md). +``` ```{literalinclude} /_includes/cwl/expressions/expression.cwl :language: cwl @@ -112,7 +113,91 @@ only in certain fields. These are: [file-prop]: https://www.commonwl.org/v1.0/CommandLineTool.html#File +## Using external libraries and inline JavaScript code with `expressionLib` + +The requirement `InlineJavascriptRequirement` supports an `expressionLib` attribute +that allows users to load external JavaScript files, or to provide inline JavaScript +code. + +Entries added to the `expressionLib` attribute are parsed with the JavaScript engine +of a CWL runner. This can be used to include external files or to create JavaScript +functions that can be called in other parts of the CWL document. + +```{note} + +The CWL standards (versions 1.0 through 1.2) [states](https://www.commonwl.org/v1.0/CommandLineTool.html#Expressions) + that the only version of JavaScript valid in CWL expressions is +[ECMAScript 5.1](https://262.ecma-international.org/5.1/). This means that any +code that you include or write in your CWL Document must be compliant with +ECMAScript 5.1. +``` + +For example, we can use `InlineJavascriptRequirement` and write a JavaScript function +inline in `expressionLib`. That function can then be used in other parts of the +CWL document: + +```{literalinclude} /_includes/cwl/expressions/hello-world-expressionlib-inline.cwl +:language: cwl +:caption: "`hello-world-expressionlib-inline.cwl`" +:name: "`hello-world-expressionlib-inline.cwl`" +:emphasize-lines: 5, 14, 32 +``` + +Running this CWL workflow will invoke the JavaScript function and result in +the `echo` command printing the input message with capital initial letters: + +```{runcmd} cwltool hello-world-expressionlib-inline.cwl --message "hello world" +:caption: "Running `hello-world-expressionlib-inline.cwl`." +:name: running-hell-world-expressionlib-inline-cwl +:working-directory: src/_includes/cwl/expressions/ +``` + +Let's move the `capitalizeWords` function to an external file, `custom-functions.js`, and +import it in our CWL document: + +```{literalinclude} /_includes/cwl/expressions/custom-functions.js +:language: javascript +:caption: "`custom-functions.js`" +:name: "`custom-functions.js`" +``` + +```{literalinclude} /_includes/cwl/expressions/hello-world-expressionlib-external.cwl +:language: cwl +:caption: "`hello-world-expressionlib-external.cwl`" +:name: "`hello-world-expressionlib-external.cwl`" +:emphasize-lines: 5-6, 14 +``` + +The `custom-functions.js` file is included in the CWL document with the `$include: custom-functions.js` +statement. That makes the functions and variables available to be used in other parts of +the CWL document. + +```{runcmd} cwltool hello-world-expressionlib-external.cwl --message "hello world" +:caption: "Running `hello-world-expressionlib-external.cwl`." +:name: running-hell-world-expressionlib-external-cwl +:working-directory: src/_includes/cwl/expressions/ +``` -% TODO -% - (maybe not before other concepts? move this to after inputs/outputs/etc?) -% - External libraries and expressionLib - https://github.com/common-workflow-language/user_guide/issues/126 +Finally, note that you can have both inline and external JavaScript code in your +CWL document. In this final example we have added another entry to the `expressionLib` +attribute with the new function `createHelloWorldMessage`, that calls the `capitalizeWords` +function from the external file `custom-functions.js`. + +```{literalinclude} /_includes/cwl/expressions/hello-world-expressionlib.cwl +:language: cwl +:caption: "`hello-world-expressionlib.cwl`" +:name: "`hello-world-expressionlib.cwl`" +:emphasize-lines: 5-17, 25 +``` + +```{runcmd} cwltool hello-world-expressionlib.cwl --message "hello world" +:caption: "Running `hello-world-expressionlib.cwl`." +:name: running-hell-world-expressionlib-cwl +:working-directory: src/_includes/cwl/expressions/ +``` + +```{note} +The `$include` statement can be used to include a file from the local disk or from a remote location. +It works with both relative and absolute paths. Read the [text about `$include`](https://www.commonwl.org/v1.0/SchemaSalad.html#Include) +from the CWL specification to learn more about it. +``` diff --git a/src/topics/file-formats.md b/src/topics/file-formats.md index 759205ee..54dee779 100644 --- a/src/topics/file-formats.md +++ b/src/topics/file-formats.md @@ -8,8 +8,8 @@ type-checking when creating parameter files. For file formats, we recommend referencing existing ontologies (like EDAM in our example), reference a local ontology for your institution, or do not add a file format initially for quick development before sharing your tool with -others. You can browse existing file format listings for IANA [here][IANA] and -for EDAM [here][EDAM]. +others. You can browse existing [IANA file format listings][IANA] and +[EDAM file format listings][EDAM] on their websites. In the next tutorial, we explain the `$namespaces` and `$schemas` section of the document in greater detail, so don't worry about these for now. @@ -42,11 +42,11 @@ parameterization. :name: sample.yml ``` -___Note:___ To follow the example below, you need to download the example input file, *file-formats.bam*. The file is available from [https://github.com/common-workflow-language/user_guide/raw/main/_includes/cwl/file-formats/file-formats.bam -](https://github.com/common-workflow-language/user_guide/raw/main/_includes/cwl/file-formats/file-formats.bam) and can be downloaded e.g. via `wget`: +___Note:___ To follow the example below, you need to download the example input file, *file-formats.bam*. The file is available from + and can be downloaded e.g. via `wget`: ```{code-block} -$ wget https://github.com/common-workflow-language/user_guide/raw/main/_includes/cwl/file-formats/file-formats.bam +$ wget https://github.com/common-workflow-language/user_guide/raw/main/src/_includes/cwl/file-formats/file-formats.bam ``` Now invoke `cwltool` with the tool description and the input object on the diff --git a/src/topics/index.md b/src/topics/index.md index 90920979..09ec4142 100644 --- a/src/topics/index.md +++ b/src/topics/index.md @@ -21,4 +21,5 @@ best-practices.md file-formats.md metadata-and-authorship.md specifying-software-requirements.md +troubleshooting.md ``` diff --git a/src/topics/inputs.md b/src/topics/inputs.md index 0ab20d05..6733f47f 100644 --- a/src/topics/inputs.md +++ b/src/topics/inputs.md @@ -29,6 +29,18 @@ Create a file called `inp-job.yml`: :name: inp-job.yml ``` +````{note} +You can use `cwltool` to create a template input object. That saves you from having +to type all the input parameters in a input object file: + +```{runcmd} cwltool --make-template inp.cwl +:working-directory: src/_includes/cwl/inputs +``` + +You can redirect the output to a file, i.e. `cwltool --make-template inp.cwl > inp-job.yml`, +and then modify the default values with your desired input values. +```` + Notice that "example_file", as a `File` type, must be provided as an object with the fields `class: File` and `path`. @@ -176,7 +188,7 @@ This will be demonstrated in the next lesson and is discussed in more detail in the [YAML Guide](yaml-guide.md#arrays). You can specify arrays of arrays, arrays of records, and other complex types. -## Advanced Inputs +## Inclusive and Exclusive Inputs Sometimes an underlying tool has several arguments that must be provided together (they are dependent) or several arguments that cannot be provided @@ -197,6 +209,7 @@ parameters together to describe these two conditions. ```{runcmd} cwltool record.cwl record-job1.yml :working-directory: src/_includes/cwl/inputs/ +:emphasize-lines: 6-7 ``` In the first example, you can't provide `itemA` without also providing `itemB`. @@ -209,6 +222,7 @@ In the first example, you can't provide `itemA` without also providing `itemB`. ```{runcmd} cwltool record.cwl record-job2.yml :working-directory: src/_includes/cwl/inputs +:emphasize-lines: 4, 10-11, 23 ```` ```{code-block} console @@ -216,8 +230,8 @@ $ cat output.txt -A one -B two -C three ``` -In the second example, `itemC` and `itemD` are exclusive, so only `itemC` -is added to the command line and `itemD` is ignored. +In the second example, `itemC` and `itemD` are exclusive, so only the first +matching item (`itemC`) is added to the command line and remaining item (`itemD`) is ignored. ```{literalinclude} /_includes/cwl/inputs/record-job3.yml :language: yaml @@ -227,6 +241,7 @@ is added to the command line and `itemD` is ignored. ```{runcmd} cwltool record.cwl record-job3.yml :working-directory: src/_includes/cwl/inputs +:emphasize-lines: 9-10, 22 ```` ```{code-block} console @@ -237,10 +252,48 @@ $ cat output.txt In the third example, only `itemD` is provided, so it appears on the command line. +### Exclusive Input Parameters with Expressions + +If you use exclusive input parameters combined with expressions, you need to be +aware that the `inputs` JavaScript object will contain one of the exclusive +input values. This means that you might need to use an **or** boolean operator +to check which values are present. + +Let's use an example that contains an exclusive `file_format` input parameter +that accepts `null` (i.e. no value provided), or any value from an enum. + +```{literalinclude} /_includes/cwl/inputs/exclusive-parameter-expressions.cwl +:language: cwl +:caption: "`exclusive-parameter-expressions.cwl`" +:name: exclusive-parameter-expressions.cwl +``` + +Note how the JavaScript expression uses the value of the exclusive input parameter +without taking into consideration a `null` value. If you provide a valid value, +such as “fasta” (one of the values of the enum), your command should execute +successfully: + +```{runcmd} cwltool exclusive-parameter-expressions.cwl --file_format fasta +:working-directory: src/_includes/cwl/inputs +```` + +However, if you do not provide any input value, then `file_format` will be +evaluated to a `null` value, which does not match the expected type for the +output field (a `string`), resulting in failure when running your workflow. + +```{runcmd} cwltool exclusive-parameter-expressions.cwl +:working-directory: src/_includes/cwl/inputs +:emphasize-lines: 5-10 +``` + +To correct it, you must remember to use an or operator in your JavaScript expression +when using exclusive parameters, or any parameter that allows `null`. For example, +the expression could be changed to `$(inputs.file_format || 'auto')`, to have +a default value if none was provided in the command line or job input file. + % TODO % % - Explain its fields, such as default, valueFrom, etc. - https://github.com/common-workflow-language/common-workflow-language/issues/359 -% - Exclusive parameters https://github.com/common-workflow-language/user_guide/issues/162 % - Optional Inputs https://github.com/common-workflow-language/user_guide/issues/44 % - Several ways of defining inputs/arguments to tools and workflows - https://github.com/common-workflow-language/user_guide/issues/33 % - Using an input output in another input - https://github.com/common-workflow-language/user_guide/issues/90 diff --git a/src/topics/specifying-software-requirements.md b/src/topics/specifying-software-requirements.md index cad04ada..c90a5ac5 100644 --- a/src/topics/specifying-software-requirements.md +++ b/src/topics/specifying-software-requirements.md @@ -1,6 +1,6 @@ # Specifying Software Requirements -Often tool descriptions will be written for a specific version of a software. To +Often, tool descriptions will be written for a specific version of a software. To make it easier for others to use your descriptions, you can include a `SoftwareRequirement` field in the `hints` section. This may also help to avoid confusion about which version of a tool the @@ -23,7 +23,7 @@ hints: ``` Depending on your CWL runner, these hints may be used to check -that required software is installed and available before the job is run. To enable +that the required software is installed and available before the job is run. To enable these checks with the reference implementation, use the [dependency resolvers configuration][dependencies]. As well as a version number, a unique resource identifier (URI) for the tool is @@ -31,8 +31,7 @@ given in the form of an [RRID][rrid]. Resources with RRIDs can be looked up in t [SciCrunch][scicrunch] registry, which provides a portal for finding, tracking, and referring to scientific resources consistently. If you want to specify a tool as a `SoftwareRequirement`, search for the tool on SciCrunch and use the -RRID that it has been assigned in the registry. (Follow [this tutorial][scicrunch-add-tool] -if you want to add a tool to SciCrunch.) You can use this RRID to refer +RRID that it has been assigned in the registry. (Follow this [Adding a Resource Tutorial][scicrunch-add-tool] to add a tool to SciCrunch). You can use this RRID to refer to the tool (via [identifiers.org][identifiers]) in the `specs` field of your requirement description. Other good choices, in order of preference, are to include the DOI for the main tool citation and the URL to the tool. diff --git a/src/topics/troubleshooting.md b/src/topics/troubleshooting.md new file mode 100644 index 00000000..d9543fa8 --- /dev/null +++ b/src/topics/troubleshooting.md @@ -0,0 +1,63 @@ +# Troubleshooting + +In this section you will find ways to troubleshoot when you have problems executing CWL. +We focus on `cwltool` here but some of these techniques may apply to other CWL Runners. + +## Run `cwltool` with `cachedir` + +You can use the `--cachedir` option when running a workflow to tell `cwltool` to +cache intermediate files (files that are not input nor output files, but created +while your workflow is running). By default, these files are created in a +temporary directory but writing them to a separate directory makes accessing +them easier. + +In the following example `troubleshooting-wf1.cwl` we have two steps, `step_a` and `step_b`. +The workflow is equivalent to `echo "Hello World" | rev`, which would print the message +"Hello World" reversed, i.e. "dlroW olleH". However, the second step, `step_b`, **has a typo**, +where instead of executing the `rev` command it tries to execute `revv`, which +fails. + +```{literalinclude} /_includes/cwl/troubleshooting/troubleshooting-wf1.cwl +:language: cwl +:name: "`troubleshooting-wf1.cwl`" +:caption: "`troubleshooting-wf1.cwl`" +:emphasize-lines: 42 +``` + +Let's execute this workflow with `/tmp/cachedir/` as the `--cachedir` value (`cwltool` will +create the directory for you if it does not exist already): + +```{runcmd} cwltool --cachedir /tmp/cachedir/ troubleshooting-wf1.cwl +:working-directory: src/_includes/cwl/troubleshooting +:emphasize-lines: 13-15, 18-20 +``` + +The workflow is in the `permanentFail` status due to `step_b` failing to execute the +non-existent `revv` command. The `step_a` was executed successfully and its output +has been cached in your `cachedir` location. You can inspect the intermediate files +created: + +```{runcmd} tree /tmp/cachedir +:emphasize-lines: 4 +``` + +Each workflow step has received a unique ID (the long value that looks like a hash). +The `${HASH}.status` files display the status of each step executed by the workflow. +And the `step_a` output file `stdout.txt` is visible in the output of the command above. + +Now fix the typo so `step_b` executes `rev` (i.e. replace `revv` by `rev` in the +`step_b`). After fixing the typo, when you execute `cwltool` with the same arguments +as the previous time, note that now `cwltool` output contains information about +pre-cached outputs for `step_a`, and about a new cache entry for the output of `step_b`. +Also note that the status of `step_b` is now of success. + +```{runcmd} cwltool --cachedir /tmp/cachedir/ troubleshooting-wf1-stepb-fixed.cwl +:working-directory: src/_includes/cwl/troubleshooting +:emphasize-lines: 12-16 +``` + +In this example the workflow step `step_a` was not re-evaluated as it had been cached, and +there was no change in its execution or output. Furthermore, `cwltool` was able to recognize +when it had to re-evaluate `step_b` after we fixed the executable name. This technique is +useful for troubleshooting your CWL documents and also as a way to prevent `cwltool` to +re-evaluate steps unnecessarily. diff --git a/src/topics/using-containers.md b/src/topics/using-containers.md index 75cdb39e..8a9551b9 100644 --- a/src/topics/using-containers.md +++ b/src/topics/using-containers.md @@ -30,7 +30,7 @@ then print "Hello World" to the standard output. :name: docker-job.yml ``` -Before we run this, lets just break it down and see what some bits do. Most of this +Before we run this, let's just break it down and see what some bits do. Most of this has been explained in previous sections, the only part that is really new is the `dockerRequirement` section. @@ -46,7 +46,7 @@ then need to specify some `hints` for how to find the container we want. In thi just our requirements for the docker container in `DockerRequirements`. The `dockerPull:` parameter takes the same value that you would pass to a `docker pull` command. That is, the name of the container image (you can even specify the tag, which is good idea for -best practises when using containers for reproducible research). In this case we have +best practices when using containers for reproducible research). In this case we have used a container called `node:slim`. Provide a "hello.js" and invoke `cwltool` providing the tool description and the diff --git a/src/topics/yaml-guide.md b/src/topics/yaml-guide.md index 070d15ef..6d05c6ed 100644 --- a/src/topics/yaml-guide.md +++ b/src/topics/yaml-guide.md @@ -44,7 +44,7 @@ The YAML above defines four keys - with their four respective values. Values can be character strings, -numeric (integer, floating point, or scientfic representation), +numeric (integer, floating point, or scientific representation), Boolean (`true` or `false`), or more complex nested types (see below). diff --git a/src/tutorials.md b/src/tutorials.md index a66adcc2..113b2368 100644 --- a/src/tutorials.md +++ b/src/tutorials.md @@ -2,9 +2,7 @@ % https://github.com/common-workflow-language/user_guide/issues/160 -This is a list to tutorials provided by the CWL community. Send a -[pull request](https://github.com/common-workflow-language/user_guide/) -if you would like to add another tutorial to the list. +This is a list of tutorials provided by the CWL community. Use the `Edit this page` link in the menu if you would like to add another tutorial to the list. ## Beginner Tutorials