From 26aee57d585e0a7c38d2cd45e37caa445e31d935 Mon Sep 17 00:00:00 2001 From: jgallowa07 Date: Mon, 6 Dec 2021 15:11:58 -0800 Subject: [PATCH 01/19] # This is a combination of 4 commits. # This is the 1st commit message: added configurations # This is the commit message #2: added config files added labels # This is the commit message #3: cleaniup # This is the commit message #4: added diagnostics - full run --- Dockerfile | 2 +- diagnostics/nextflow_report.html.9 | 1061 ++++++++++++++++++++++++++++ main.nf | 7 +- modules/macro_flu.nf | 7 + modules/micro.nf | 10 +- modules/treetime_validation.nf | 5 +- nextflow.rhino.config | 30 + run_rhino.sh | 15 + 8 files changed, 1129 insertions(+), 8 deletions(-) create mode 100644 diagnostics/nextflow_report.html.9 create mode 100644 nextflow.rhino.config create mode 100755 run_rhino.sh diff --git a/Dockerfile b/Dockerfile index 523da04..936180e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -78,4 +78,4 @@ RUN git clone --depth 1 https://github.com/4ment/phylostan /phylostan RUN cd /phylostan && pip install . && phylostan --help RUN echo "source activate bito" > ~/.bashrc -ENV PATH /opt/conda/envs/bito/bin:$PATH \ No newline at end of file +ENV PATH /opt/conda/envs/bito/bin:$PATH diff --git a/diagnostics/nextflow_report.html.9 b/diagnostics/nextflow_report.html.9 new file mode 100644 index 0000000..20545e9 --- /dev/null +++ b/diagnostics/nextflow_report.html.9 @@ -0,0 +1,1061 @@ + + + + + + + + + + + [mighty_mcnulty] Nextflow Workflow Report + + + + + + + +
+
+ +

Nextflow workflow report

+

[mighty_mcnulty]

+ + +
+

Workflow execution completed unsuccessfully!

+

The exit status of the task that caused the workflow execution to fail was: 2.

+

The full error message was:

+ 
Error executing process > 'treetime_validation:TREETIME_VALIDATION_SUBTREES (13)'
+
+Caused by:
+  Process `treetime_validation:TREETIME_VALIDATION_SUBTREES (13)` terminated with an error exit status (2)
+
+Command executed:
+
+  python2.7 /fh/fast/matsen_e/shared/autodiff-experiments/treetime_validation/generate_flu_subtrees_dataset_run.py --size 20                                                            --out_dir dataset                                                            --suffix 3                                                            --treetime_file treetime_res.csv                                                            --lsd_file lsd_res.csv                                                            --aln_file /fh/fast/matsen_e/shared/autodiff-experiments/treetime_validation/resources/flu_H3N2/H3N2_HA_2011_2013.fasta                                                            --tree_file /fh/fast/matsen_e/shared/autodiff-experiments/treetime_validation/resources/flu_H3N2/H3N2_HA_2011_2013.nwk                                                            --template_file /fh/fast/matsen_e/shared/autodiff-experiments/treetime_validation/resources/beast/template_bedford_et_al_2015.xml
+
+Command exit status:
+  2
+
+Command output:
+  (empty)
+
+Command error:
+  python2.7: can't open file '/fh/fast/matsen_e/shared/autodiff-experiments/treetime_validation/generate_flu_subtrees_dataset_run.py': [Errno 2] No such file or directory
+
+Work dir:
+  /fh/fast/matsen_e/shared/autodiff-experiments/r_output/work/a7/0dfd5acb70ab96f2e07344f04ed719
+
+Tip: you can replicate the issue by changing to the process work dir and entering the command `bash .command.run`
+
+ + +
+
Run times
+
+ 06-Dec-2021 15:08:12 - 06-Dec-2021 16:34:17 + (duration: 1h 26m 5s) +
+ +
+
+
  0 succeeded  
+
  0 cached  
+
  0 ignored  
+
  1 failed  
+
+
+ +
Nextflow command
+ 
nextflow -C ./nextflow.rhino.config run main.nf -with-report ./r_output/nextflow_report.html -work-dir ./r_output/work/ -resume
+
+ +
+
CPU-Hours
+
(a few seconds)
+ +
Launch directory
+
/fh/fast/matsen_e/shared/autodiff-experiments
+ +
Work directory
+
/fh/fast/matsen_e/shared/autodiff-experiments/r_output/work
+ +
Project directory
+
/fh/fast/matsen_e/shared/autodiff-experiments
+ + +
Script name
+
main.nf
+ + + +
Script ID
+
828e3ad1d0fe1064c092b57144587009
+ + +
Workflow session
+
b0e6695c-e5a6-4443-b873-f2b81538afb2
+ + + +
Workflow profile
+
standard
+ + + +
Nextflow version
+
version 21.04.3, build 5560 (21-07-2021 15:09 UTC)
+
+
+
+ +
+

Resource Usage

+

These plots give an overview of the distribution of resource usage for each process.

+ +

CPU

+ +
+
+
+
+
+
+
+ +
+ +

Memory

+ +
+
+
+
+
+
+
+
+
+
+
+ +

Job Duration

+ +
+
+
+
+
+
+
+
+ +

I/O

+ +
+
+
+
+
+
+
+
+
+ +
+
+

Tasks

+

This table shows information about each task in the workflow. Use the search box on the right + to filter rows for specific values. Clicking headers will sort the table by that value and + scrolling side to side will reveal more columns.

+
+ + +
+
+
+
+
+ +
+ (tasks table omitted because the dataset is too big) +
+
+ + + + + + + diff --git a/main.nf b/main.nf index bab8fc3..fd74496 100644 --- a/main.nf +++ b/main.nf @@ -11,6 +11,7 @@ include { micro } from "./modules/micro.nf" include { macro_flu } from "./modules/macro_flu.nf" process RUN_LSD { + label 'auto_diff_exp' input: tuple val(size), val(rep), @@ -32,6 +33,7 @@ process RUN_LSD { } process CONVERT_LSD_NEXUS_TO_NEWICK { + label 'auto_diff_exp' input: tuple val(size), val(rep), path(lsd_nexus) output: @@ -55,6 +57,7 @@ def group_per_size_rep(newick_ch, create_sub_ch) { } process CREATE_SUB_FILES { + label 'auto_diff_exp' input: tuple val(size), val(rep), path(lsd_dates), path(newick_file) @@ -89,7 +92,7 @@ workflow { CONVERT_LSD_NEXUS_TO_NEWICK.out, CREATE_SUB_FILES.out) - macro_flu(data) + // macro_flu(data) micro(data.map { it.take(4) }) -} \ No newline at end of file +} diff --git a/modules/macro_flu.nf b/modules/macro_flu.nf index 200b7bb..fd279c6 100644 --- a/modules/macro_flu.nf +++ b/modules/macro_flu.nf @@ -14,6 +14,7 @@ torchtree_jc69_template = "$flu_H3N2/phylotorch-JC69.template" process COMPILE_PHYLOSTAN { + label 'auto_diff_exp' input: val(name) val(model) @@ -33,6 +34,7 @@ process COMPILE_PHYLOSTAN { process RUN_PHYLOSTAN { publishDir "$params.results/macro/phylostan", mode: 'copy' + label 'auto_diff_exp' input: tuple val(size), val(rep), path(tree_file), path(seq_file) @@ -62,6 +64,7 @@ process RUN_PHYLOSTAN { } process PREPARE_PHYSHER { + label 'auto_diff_exp' input: tuple val(size), val(rep), path(lsd_newick), path(seq_file), path(lsd_dates) @@ -79,6 +82,7 @@ process PREPARE_PHYSHER { process RUN_PHYSHER { publishDir "$params.results/macro/physher", mode: 'copy' + label 'auto_diff_exp' input: tuple val(size), val(rep), path(lsd_newick), path(seq_file), path(physher_json) @@ -91,6 +95,7 @@ process RUN_PHYSHER { } process PREPARE_TORCHTREE { + label 'auto_diff_exp' label 'bito' input: @@ -109,6 +114,7 @@ process PREPARE_TORCHTREE { } process RUN_TORCHTREE { + label 'auto_diff_exp' label 'bito' publishDir "$params.results/macro/torchtree", mode: 'copy' @@ -125,6 +131,7 @@ process RUN_TORCHTREE { } process RUN_PHYLOJAX { + label 'auto_diff_exp' label 'bito' publishDir "$params.results/macro/phylojax", mode: 'copy' diff --git a/modules/micro.nf b/modules/micro.nf index 5e39928..031fca8 100644 --- a/modules/micro.nf +++ b/modules/micro.nf @@ -8,6 +8,7 @@ params.results = "results" phylox = Channel.of("torchtree", "bitorch", "phylojax") process RUN_PHYSHER_BENCHMARK { + label 'auto_diff_exp' publishDir "$params.results/micro/physher", mode: 'copy' input: @@ -27,6 +28,7 @@ process RUN_PHYSHER_BENCHMARK { } process RUN_PHYLOX_BENCHMARK { + label 'auto_diff_exp' label 'bito' publishDir "$params.results/micro/${phylox}", mode: 'copy' @@ -54,6 +56,7 @@ process RUN_PHYLOX_BENCHMARK { } process COMBIME_CSV { + label 'auto_diff_exp' publishDir "$params.results/micro/", mode: 'copy' input: @@ -62,8 +65,9 @@ process COMBIME_CSV { path("micro.csv") """ - head -n1 ${files[0]} > micro.csv - tail -q -n+2 *.csv >> micro.csv + head -n1 ${files[0]} > micro + tail -q -n+2 *.csv >> micro + mv micro micro.csv """ } @@ -81,4 +85,4 @@ workflow micro { RUN_PHYSHER_BENCHMARK.out.collect(), RUN_PHYLOX_BENCHMARK.out.collect()) COMBIME_CSV(ch_files.collect()) -} \ No newline at end of file +} diff --git a/modules/treetime_validation.nf b/modules/treetime_validation.nf index 8694db2..2fac110 100644 --- a/modules/treetime_validation.nf +++ b/modules/treetime_validation.nf @@ -2,7 +2,7 @@ nextflow.enable.dsl = 2 -params.subtrees = Channel.of(20, 50, 100, 200, 500, 750, 1000, 1250, 1500, 2000) +params.subtrees = Channel.of(20, 50, 100, 200, 500, 750, 1000)//, 1250, 1500, 2000) params.subtrees_replicates = Channel.of(0..9) if(params.enable_beast){ @@ -19,6 +19,7 @@ beast_template = "$params.base/resources/beast/template_bedford_et_al_2015.xml" treetime_flu_H3N2 = "$params.base/flu_H3N2/subtree_samples" process TREETIME_VALIDATION_SUBTREES { + label 'auto_diff_exp' label 'treetime' publishDir "${treetime_flu_H3N2}", mode: 'copy' @@ -52,4 +53,4 @@ workflow treetime_validation { params.subtrees.combine(params.subtrees_replicates) | TREETIME_VALIDATION_SUBTREES emit: TREETIME_VALIDATION_SUBTREES.out[0] -} \ No newline at end of file +} diff --git a/nextflow.rhino.config b/nextflow.rhino.config new file mode 100644 index 0000000..656474b --- /dev/null +++ b/nextflow.rhino.config @@ -0,0 +1,30 @@ +manifest { + name = 'autodiff-experiments' + author = 'Mathieu Fourment' + homePage = 'https://github.com/4ment/autodiff-experiments' + description = 'Benchmarking of automatic and analytical differentiation' + mainScript = 'main.nf' +} + +// COMPUTATIONAL PLATFORM SETUP +profiles { + standard { + process { + executor = 'slurm' + withLabel: auto_diff_exp { + container = 'quay.io/matsengrp/autodiff-experiments' + queue = 'campus-new' + cpus = 24 + memory = 16.GB + //time = '2-5' + //clusterOptions = '-A matsen_e' + } + } + singularity { + enabled = true + autoMounts = true + cacheDir = 'temp/containers/' + runOptions = '--contain -W /tmp/' + } + } +} diff --git a/run_rhino.sh b/run_rhino.sh new file mode 100755 index 0000000..8dec194 --- /dev/null +++ b/run_rhino.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e +source /app/lmod/lmod/init/profile + +module load nextflow +module load Singularity +export PATH=$SINGULARITYROOT/bin/:$PATH + +/usr/bin/time nextflow \ + -C ./nextflow.rhino.config \ + run main.nf \ + -with-report ./r_output/nextflow_report.html \ + -work-dir ./r_output/work/ \ + -resume From afeaf8da5e18b36081db70020107452fb163f80c Mon Sep 17 00:00:00 2001 From: jgallowa07 Date: Mon, 13 Dec 2021 07:45:32 -0800 Subject: [PATCH 02/19] cleanup and rebase --- main.nf | 3 - modules/macro_flu.nf | 7 --- modules/micro.nf | 3 - modules/treetime_validation.nf | 1 - nextflow.rhino.config | 15 +++-- .../nextflow_report.html | 58 +++++++------------ 6 files changed, 27 insertions(+), 60 deletions(-) rename diagnostics/nextflow_report.html.9 => r_output/nextflow_report.html (78%) diff --git a/main.nf b/main.nf index fd74496..0b8920f 100644 --- a/main.nf +++ b/main.nf @@ -11,7 +11,6 @@ include { micro } from "./modules/micro.nf" include { macro_flu } from "./modules/macro_flu.nf" process RUN_LSD { - label 'auto_diff_exp' input: tuple val(size), val(rep), @@ -33,7 +32,6 @@ process RUN_LSD { } process CONVERT_LSD_NEXUS_TO_NEWICK { - label 'auto_diff_exp' input: tuple val(size), val(rep), path(lsd_nexus) output: @@ -57,7 +55,6 @@ def group_per_size_rep(newick_ch, create_sub_ch) { } process CREATE_SUB_FILES { - label 'auto_diff_exp' input: tuple val(size), val(rep), path(lsd_dates), path(newick_file) diff --git a/modules/macro_flu.nf b/modules/macro_flu.nf index fd279c6..200b7bb 100644 --- a/modules/macro_flu.nf +++ b/modules/macro_flu.nf @@ -14,7 +14,6 @@ torchtree_jc69_template = "$flu_H3N2/phylotorch-JC69.template" process COMPILE_PHYLOSTAN { - label 'auto_diff_exp' input: val(name) val(model) @@ -34,7 +33,6 @@ process COMPILE_PHYLOSTAN { process RUN_PHYLOSTAN { publishDir "$params.results/macro/phylostan", mode: 'copy' - label 'auto_diff_exp' input: tuple val(size), val(rep), path(tree_file), path(seq_file) @@ -64,7 +62,6 @@ process RUN_PHYLOSTAN { } process PREPARE_PHYSHER { - label 'auto_diff_exp' input: tuple val(size), val(rep), path(lsd_newick), path(seq_file), path(lsd_dates) @@ -82,7 +79,6 @@ process PREPARE_PHYSHER { process RUN_PHYSHER { publishDir "$params.results/macro/physher", mode: 'copy' - label 'auto_diff_exp' input: tuple val(size), val(rep), path(lsd_newick), path(seq_file), path(physher_json) @@ -95,7 +91,6 @@ process RUN_PHYSHER { } process PREPARE_TORCHTREE { - label 'auto_diff_exp' label 'bito' input: @@ -114,7 +109,6 @@ process PREPARE_TORCHTREE { } process RUN_TORCHTREE { - label 'auto_diff_exp' label 'bito' publishDir "$params.results/macro/torchtree", mode: 'copy' @@ -131,7 +125,6 @@ process RUN_TORCHTREE { } process RUN_PHYLOJAX { - label 'auto_diff_exp' label 'bito' publishDir "$params.results/macro/phylojax", mode: 'copy' diff --git a/modules/micro.nf b/modules/micro.nf index 031fca8..3340213 100644 --- a/modules/micro.nf +++ b/modules/micro.nf @@ -8,7 +8,6 @@ params.results = "results" phylox = Channel.of("torchtree", "bitorch", "phylojax") process RUN_PHYSHER_BENCHMARK { - label 'auto_diff_exp' publishDir "$params.results/micro/physher", mode: 'copy' input: @@ -28,7 +27,6 @@ process RUN_PHYSHER_BENCHMARK { } process RUN_PHYLOX_BENCHMARK { - label 'auto_diff_exp' label 'bito' publishDir "$params.results/micro/${phylox}", mode: 'copy' @@ -56,7 +54,6 @@ process RUN_PHYLOX_BENCHMARK { } process COMBIME_CSV { - label 'auto_diff_exp' publishDir "$params.results/micro/", mode: 'copy' input: diff --git a/modules/treetime_validation.nf b/modules/treetime_validation.nf index 2fac110..d901d72 100644 --- a/modules/treetime_validation.nf +++ b/modules/treetime_validation.nf @@ -19,7 +19,6 @@ beast_template = "$params.base/resources/beast/template_bedford_et_al_2015.xml" treetime_flu_H3N2 = "$params.base/flu_H3N2/subtree_samples" process TREETIME_VALIDATION_SUBTREES { - label 'auto_diff_exp' label 'treetime' publishDir "${treetime_flu_H3N2}", mode: 'copy' diff --git a/nextflow.rhino.config b/nextflow.rhino.config index 656474b..159ccfb 100644 --- a/nextflow.rhino.config +++ b/nextflow.rhino.config @@ -11,14 +11,13 @@ profiles { standard { process { executor = 'slurm' - withLabel: auto_diff_exp { - container = 'quay.io/matsengrp/autodiff-experiments' - queue = 'campus-new' - cpus = 24 - memory = 16.GB - //time = '2-5' - //clusterOptions = '-A matsen_e' - } + //withLabel: auto_diff_exp { + container = 'quay.io/matsengrp/autodiff-experiments' + queue = 'campus-new' + cpus = 24 + memory = 16.GB + //clusterOptions = '-A matsen_e' + //} } singularity { enabled = true diff --git a/diagnostics/nextflow_report.html.9 b/r_output/nextflow_report.html similarity index 78% rename from diagnostics/nextflow_report.html.9 rename to r_output/nextflow_report.html index 20545e9..7545f9d 100644 --- a/diagnostics/nextflow_report.html.9 +++ b/r_output/nextflow_report.html @@ -18,11 +18,11 @@ - + - [mighty_mcnulty] Nextflow Workflow Report + [ridiculous_williams] Nextflow Workflow Report - - - - - -
-
- -

Nextflow workflow report

-

[ridiculous_williams] (resumed run)

- - -
- Workflow execution completed successfully! -
- - -
-
Run times
-
- 13-Dec-2021 07:27:55 - 13-Dec-2021 07:28:02 - (duration: 7.4s) -
- -
-
-
  0 succeeded  
-
  631 cached  
-
  0 ignored  
-
  0 failed  
-
-
- -
Nextflow command
- 
nextflow -C ./nextflow.rhino.config run main.nf -with-report ./r_output/nextflow_report.html -work-dir ./r_output/work/ -resume
-
- -
-
CPU-Hours
-
442.3 (100% cached)
- -
Launch directory
-
/fh/fast/matsen_e/shared/autodiff-experiments
- -
Work directory
-
/fh/fast/matsen_e/shared/autodiff-experiments/r_output/work
- -
Project directory
-
/fh/fast/matsen_e/shared/autodiff-experiments
- - -
Script name
-
main.nf
- - - -
Script ID
-
f2a376c548be629be536268b614dab9d
- - -
Workflow session
-
d9817ff2-ddb2-4582-ac32-2b13ad0f41c0
- - - -
Workflow profile
-
standard
- - -
Workflow container
-
quay.io/matsengrp/autodiff-experiments
- -
Container engine
-
singularity
- - -
Nextflow version
-
version 21.04.3, build 5560 (21-07-2021 15:09 UTC)
-
-
-
- -
-

Resource Usage

-

These plots give an overview of the distribution of resource usage for each process.

- -

CPU

- -
-
-
-
-
-
-
- -
- -

Memory

- -
-
-
-
-
-
-
-
-
-
-
- -

Job Duration

- -
-
-
-
-
-
-
-
- -

I/O

- -
-
-
-
-
-
-
-
-
- -
-
-

Tasks

-

This table shows information about each task in the workflow. Use the search box on the right - to filter rows for specific values. Clicking headers will sort the table by that value and - scrolling side to side will reveal more columns.

-
- - -
-
-
-
-
- -
- (tasks table omitted because the dataset is too big) -
-
- -
-
- Generated by Nextflow, version 21.04.3 -
-
- - - - - diff --git a/run_gng.sh b/run_gng.sh index 62e8755..f2cf15c 100755 --- a/run_gng.sh +++ b/run_gng.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -source /app/lmod/lmod/init/profile +# source /app/lmod/lmod/init/profile module load nextflow module load Singularity @@ -14,5 +14,4 @@ export PATH=$SINGULARITYROOT/bin/:$PATH -profile standard \ -with-report "$(date -I)-gng-results"/nextflow_report.html \ -with-trace "$(date -I)-gng-results"/trace.txt \ - -work-dir ./gng_work/ \ - -resume + -work-dir '/fh/scratch/delete30/matsen_e/mathieu/temp/gng_work/' diff --git a/run_rhino.sh b/run_rhino.sh index f3622ae..951823a 100755 --- a/run_rhino.sh +++ b/run_rhino.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -source /app/lmod/lmod/init/profile +# source /app/lmod/lmod/init/profile module load nextflow module load Singularity @@ -10,9 +10,9 @@ export PATH=$SINGULARITYROOT/bin/:$PATH /usr/bin/time nextflow \ -C ./configs/rhino.config \ run main.nf \ - --results "$(date -I)-rhino-results" \ + --results "batch-results" \ -profile rhino \ - -with-report ./r_output/nextflow_report.html \ - -work-dir ./r_output/work/ \ - -with-trace \ + -with-report "batch-results"/nextflow_report.html \ + -with-trace "batch-results"/trace.txt \ + -work-dir "batch-results/work/" \ -resume diff --git a/scripts/parse-trace.py b/scripts/parse-trace.py index 3a43817..29f983a 100644 --- a/scripts/parse-trace.py +++ b/scripts/parse-trace.py @@ -13,8 +13,12 @@ with open(sys.argv[2], "r") as fp: for line in fp: a = line.split("\t") + if 'FAILED' in a or 'ABORTED' in a: + continue if "task_id" == a[0]: print("program\tsize\treplicate\t" + line, end="") + if a[1] not in dic: + continue elif "macro_flu:RUN_" in a[3]: for f in dic[a[1]]: if ".txt" in f: From 3c96fe993f36eaf43ce0068f365ea70d9e4c85f2 Mon Sep 17 00:00:00 2001 From: jgallowa07 Date: Wed, 7 Sep 2022 16:27:31 -0700 Subject: [PATCH 07/19] relavant run script --- run_gng.sh | 17 ----------------- run_quoll.sh | 17 ----------------- 2 files changed, 34 deletions(-) delete mode 100755 run_gng.sh delete mode 100755 run_quoll.sh diff --git a/run_gng.sh b/run_gng.sh deleted file mode 100755 index f2cf15c..0000000 --- a/run_gng.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -e -# source /app/lmod/lmod/init/profile - -module load nextflow -module load Singularity -export PATH=$SINGULARITYROOT/bin/:$PATH - -/usr/bin/time nextflow \ - -C ./configs/rhino.config \ - run main.nf \ - --results "$(date -I)-gng-results" \ - -profile standard \ - -with-report "$(date -I)-gng-results"/nextflow_report.html \ - -with-trace "$(date -I)-gng-results"/trace.txt \ - -work-dir '/fh/scratch/delete30/matsen_e/mathieu/temp/gng_work/' diff --git a/run_quoll.sh b/run_quoll.sh deleted file mode 100755 index fc12dd5..0000000 --- a/run_quoll.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -set -e -source /app/lmod/lmod/init/profile - -module load nextflow -module load Singularity -export PATH=$SINGULARITYROOT/bin/:$PATH - -/usr/bin/time nextflow \ - -C ./configs/rhino.config \ - run main.nf \ - --results "$(date -I)-quoll-results" \ - -profile standard \ - -with-report "$(date -I)-quoll-results"/nextflow_report.html \ - -with-trace "$(date -I)-quoll-results"/trace.txt \ - -work-dir ./q_work/ From b6de01b8a6ec259cf768df74cd741281a43873ef Mon Sep 17 00:00:00 2001 From: jgallowa07 Date: Thu, 29 Sep 2022 08:53:35 -0700 Subject: [PATCH 08/19] update dockerfile with new treeflow --- Dockerfile | 2 ++ run_rhino.sh | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 89042e4..9f9e1de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM continuumio/anaconda3 LABEL "author"="Mathieu Fourment" LABEL "company"="University of Technology Sydney" +CACHE OFF RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -68,6 +69,7 @@ RUN ln -s /phylojax/benchmarks/benchmark.py /usr/local/bin/phylojax-benchmark \ RUN pip install phylostan==1.0.5 && phylostan --help +ADD https://api.github.com/repos/christiaanjs/treeflow/git/refs/heads/master version.json RUN git clone --depth 1 https://github.com/christiaanjs/treeflow.git /treeflow RUN cd /treeflow && /opt/conda/envs/bito/bin/pip install . RUN . /opt/conda/etc/profile.d/conda.sh && conda activate bito && treeflow_benchmark --help diff --git a/run_rhino.sh b/run_rhino.sh index 951823a..8c139cf 100755 --- a/run_rhino.sh +++ b/run_rhino.sh @@ -10,9 +10,9 @@ export PATH=$SINGULARITYROOT/bin/:$PATH /usr/bin/time nextflow \ -C ./configs/rhino.config \ run main.nf \ - --results "batch-results" \ + --results "batch-results-$(date -I)" \ -profile rhino \ - -with-report "batch-results"/nextflow_report.html \ - -with-trace "batch-results"/trace.txt \ + -with-report "batch-results-$(date -I)"/nextflow_report.html \ + -with-trace "batch-results-$(date -I)"/trace.txt \ -work-dir "batch-results/work/" \ -resume From d3dbb0e582989045ce78ba2a71daea9145915204 Mon Sep 17 00:00:00 2001 From: jgallowa07 Date: Thu, 29 Sep 2022 09:22:22 -0700 Subject: [PATCH 09/19] being treeflow to bleeding edge --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9f9e1de..50c45b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,6 @@ FROM continuumio/anaconda3 LABEL "author"="Mathieu Fourment" LABEL "company"="University of Technology Sydney" -CACHE OFF RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -69,9 +68,8 @@ RUN ln -s /phylojax/benchmarks/benchmark.py /usr/local/bin/phylojax-benchmark \ RUN pip install phylostan==1.0.5 && phylostan --help -ADD https://api.github.com/repos/christiaanjs/treeflow/git/refs/heads/master version.json RUN git clone --depth 1 https://github.com/christiaanjs/treeflow.git /treeflow -RUN cd /treeflow && /opt/conda/envs/bito/bin/pip install . +RUN cd /treeflow && git checkout e3414dc && /opt/conda/envs/bito/bin/pip install . RUN . /opt/conda/etc/profile.d/conda.sh && conda activate bito && treeflow_benchmark --help RUN echo "source activate bito" > ~/.bashrc From c639157b7f09e7e2572a9c5150fc2be21b0ce05f Mon Sep 17 00:00:00 2001 From: jgallowa07 Date: Thu, 29 Sep 2022 09:35:56 -0700 Subject: [PATCH 10/19] use full SHA in Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 50c45b7..a237bee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -69,7 +69,7 @@ RUN ln -s /phylojax/benchmarks/benchmark.py /usr/local/bin/phylojax-benchmark \ RUN pip install phylostan==1.0.5 && phylostan --help RUN git clone --depth 1 https://github.com/christiaanjs/treeflow.git /treeflow -RUN cd /treeflow && git checkout e3414dc && /opt/conda/envs/bito/bin/pip install . +RUN cd /treeflow && git checkout e3414dcc9e764d06abc3e19c1d0f55110499e2ea && /opt/conda/envs/bito/bin/pip install . RUN . /opt/conda/etc/profile.d/conda.sh && conda activate bito && treeflow_benchmark --help RUN echo "source activate bito" > ~/.bashrc From 5640e8d277562be89c1569ad1d75afd8790472ca Mon Sep 17 00:00:00 2001 From: jgallowa07 Date: Thu, 29 Sep 2022 09:53:45 -0700 Subject: [PATCH 11/19] bust cache --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a237bee..87693e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -68,8 +68,10 @@ RUN ln -s /phylojax/benchmarks/benchmark.py /usr/local/bin/phylojax-benchmark \ RUN pip install phylostan==1.0.5 && phylostan --help +# workaround to bust cache if the master branch has been updated ... +ADD https://api.github.com/repos/christiaanjs/treeflow/git/refs/heads/master /.git-hashref RUN git clone --depth 1 https://github.com/christiaanjs/treeflow.git /treeflow -RUN cd /treeflow && git checkout e3414dcc9e764d06abc3e19c1d0f55110499e2ea && /opt/conda/envs/bito/bin/pip install . +RUN cd /treeflow && /opt/conda/envs/bito/bin/pip install . RUN . /opt/conda/etc/profile.d/conda.sh && conda activate bito && treeflow_benchmark --help RUN echo "source activate bito" > ~/.bashrc From dd6394298ff1f40fa785952f9b4fc5c1fd435b72 Mon Sep 17 00:00:00 2001 From: jgallowa07 Date: Thu, 29 Sep 2022 10:15:10 -0700 Subject: [PATCH 12/19] bust entire cache and rebuild entire container --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 87693e6..f4aa560 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,9 @@ FROM continuumio/anaconda3 LABEL "author"="Mathieu Fourment" LABEL "company"="University of Technology Sydney" +# bust cache and re-run all +ADD http://date.jsontest.com /etc/builddate + RUN apt-get update && \ apt-get install -y --no-install-recommends \ autoconf \ From d0a46667db67e685a10a225f9e0b5579183734a9 Mon Sep 17 00:00:00 2001 From: jgallowa07 Date: Mon, 17 Oct 2022 12:02:24 -0700 Subject: [PATCH 13/19] latest changes for rebase --- modules/macro_flu.nf | 2 ++ run_rhino.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/macro_flu.nf b/modules/macro_flu.nf index 381156c..1d21703 100644 --- a/modules/macro_flu.nf +++ b/modules/macro_flu.nf @@ -184,6 +184,8 @@ process RUN_PHYLOJAX { process RUN_TREEFLOW { label 'bito' + errorStrategy 'ignore' + publishDir "$params.results/macro/treeflow", mode: 'copy' input: diff --git a/run_rhino.sh b/run_rhino.sh index 8c139cf..b98c6bc 100755 --- a/run_rhino.sh +++ b/run_rhino.sh @@ -14,5 +14,5 @@ export PATH=$SINGULARITYROOT/bin/:$PATH -profile rhino \ -with-report "batch-results-$(date -I)"/nextflow_report.html \ -with-trace "batch-results-$(date -I)"/trace.txt \ - -work-dir "batch-results/work/" \ + -work-dir "batch-results-$(date -I)/work/" \ -resume From 401d0af3b52280067e38924a493d863ff774af4a Mon Sep 17 00:00:00 2001 From: jgallowa07 Date: Tue, 25 Oct 2022 05:23:16 -0700 Subject: [PATCH 14/19] remove cache bust, re-install all --- Dockerfile | 2 +- configs/rhino.config | 2 +- run_rhino.sh | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index f4aa560..196c3d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ LABEL "author"="Mathieu Fourment" LABEL "company"="University of Technology Sydney" # bust cache and re-run all -ADD http://date.jsontest.com /etc/builddate +# ADD http://date.jsontest.com /etc/builddate RUN apt-get update && \ apt-get install -y --no-install-recommends \ diff --git a/configs/rhino.config b/configs/rhino.config index bcccd7e..7b9d29f 100644 --- a/configs/rhino.config +++ b/configs/rhino.config @@ -32,7 +32,7 @@ profiles { //scratch = '/fh/scratch/delete30/matsen_e/mathieu/temp/nf-scratch/' queue = 'campus-new' cpus = 4 - memory = 20.GB + memory = 32.GB container = 'quay.io/matsengrp/autodiff-experiments:phylo_patch' time = '200h' diff --git a/run_rhino.sh b/run_rhino.sh index b98c6bc..cb6870e 100755 --- a/run_rhino.sh +++ b/run_rhino.sh @@ -14,5 +14,5 @@ export PATH=$SINGULARITYROOT/bin/:$PATH -profile rhino \ -with-report "batch-results-$(date -I)"/nextflow_report.html \ -with-trace "batch-results-$(date -I)"/trace.txt \ - -work-dir "batch-results-$(date -I)/work/" \ - -resume + -work-dir "batch-results-$(date -I)/work/" #\ + #-resume From 239476acd4e9e228455bf2c210b8ba1806f51d00 Mon Sep 17 00:00:00 2001 From: jgallowa07 Date: Tue, 25 Oct 2022 15:40:52 -0700 Subject: [PATCH 15/19] bust cache and rebuild image --- Dockerfile | 3 ++- modules/macro_flu.nf | 5 +++-- run_rhino.sh | 4 ++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 196c3d7..96e50f0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ LABEL "author"="Mathieu Fourment" LABEL "company"="University of Technology Sydney" # bust cache and re-run all -# ADD http://date.jsontest.com /etc/builddate +ADD http://date.jsontest.com /etc/builddate RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -70,6 +70,7 @@ RUN ln -s /phylojax/benchmarks/benchmark.py /usr/local/bin/phylojax-benchmark \ && chmod +x /usr/local/bin/phylojax-benchmark RUN pip install phylostan==1.0.5 && phylostan --help +RUN pip install tqdm # workaround to bust cache if the master branch has been updated ... ADD https://api.github.com/repos/christiaanjs/treeflow/git/refs/heads/master /.git-hashref diff --git a/modules/macro_flu.nf b/modules/macro_flu.nf index 1d21703..d77de89 100644 --- a/modules/macro_flu.nf +++ b/modules/macro_flu.nf @@ -189,13 +189,14 @@ process RUN_TREEFLOW { publishDir "$params.results/macro/treeflow", mode: 'copy' input: - tuple val(size), val(rep), path(tree_file), path(seq_file) + tuple val(size), val(rep), path(tree_file), val(rate), path(seq_file) output: tuple path("treeflow.${size}.${rep}.txt"), path("treeflow.${size}.${rep}.log") """ { time \ treeflow_vi -i ${seq_file} \ -t ${tree_file} \ + --init-values clock_rate=${rate} \ -n ${params.iterations} > treeflow.${size}.${rep}.txt ; } 2> treeflow.${size}.${rep}.log """ } @@ -224,7 +225,7 @@ workflow macro_flu { RUN_TORCHTREE(data_run.join(PREPARE_TORCHTREE.out, by: [0, 1])) - RUN_TREEFLOW(data_run) + RUN_TREEFLOW(data.map {it.take(5)} ) ch_files = Channel.empty() ch_files = ch_files.mix( diff --git a/run_rhino.sh b/run_rhino.sh index cb6870e..b98c6bc 100755 --- a/run_rhino.sh +++ b/run_rhino.sh @@ -14,5 +14,5 @@ export PATH=$SINGULARITYROOT/bin/:$PATH -profile rhino \ -with-report "batch-results-$(date -I)"/nextflow_report.html \ -with-trace "batch-results-$(date -I)"/trace.txt \ - -work-dir "batch-results-$(date -I)/work/" #\ - #-resume + -work-dir "batch-results-$(date -I)/work/" \ + -resume From f81e9197a7c3693d2a8eea0aec9bdbfd8054add7 Mon Sep 17 00:00:00 2001 From: jgallowa07 Date: Tue, 25 Oct 2022 20:12:48 -0700 Subject: [PATCH 16/19] roll back treeflow --- Dockerfile | 4 ++-- run_rhino.sh | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 96e50f0..0a3b3d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ LABEL "author"="Mathieu Fourment" LABEL "company"="University of Technology Sydney" # bust cache and re-run all -ADD http://date.jsontest.com /etc/builddate +# ADD http://date.jsontest.com /etc/builddate RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -75,7 +75,7 @@ RUN pip install tqdm # workaround to bust cache if the master branch has been updated ... ADD https://api.github.com/repos/christiaanjs/treeflow/git/refs/heads/master /.git-hashref RUN git clone --depth 1 https://github.com/christiaanjs/treeflow.git /treeflow -RUN cd /treeflow && /opt/conda/envs/bito/bin/pip install . +RUN cd /treeflow && git checkout e3414dcc9e764d06abc3e19c1d0f55110499e2ea && /opt/conda/envs/bito/bin/pip install . RUN . /opt/conda/etc/profile.d/conda.sh && conda activate bito && treeflow_benchmark --help RUN echo "source activate bito" > ~/.bashrc diff --git a/run_rhino.sh b/run_rhino.sh index b98c6bc..505b7b0 100755 --- a/run_rhino.sh +++ b/run_rhino.sh @@ -14,5 +14,4 @@ export PATH=$SINGULARITYROOT/bin/:$PATH -profile rhino \ -with-report "batch-results-$(date -I)"/nextflow_report.html \ -with-trace "batch-results-$(date -I)"/trace.txt \ - -work-dir "batch-results-$(date -I)/work/" \ - -resume + -work-dir "batch-results-$(date -I)/work/" From 4c1125b2c42ca6700ae9ee9a3635f5e2eee1ced6 Mon Sep 17 00:00:00 2001 From: jgallowa07 Date: Tue, 25 Oct 2022 21:15:41 -0700 Subject: [PATCH 17/19] patch dockerfile --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0a3b3d9..ddbc045 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ LABEL "author"="Mathieu Fourment" LABEL "company"="University of Technology Sydney" # bust cache and re-run all -# ADD http://date.jsontest.com /etc/builddate +ADD http://date.jsontest.com /etc/builddate RUN apt-get update && \ apt-get install -y --no-install-recommends \ @@ -73,7 +73,7 @@ RUN pip install phylostan==1.0.5 && phylostan --help RUN pip install tqdm # workaround to bust cache if the master branch has been updated ... -ADD https://api.github.com/repos/christiaanjs/treeflow/git/refs/heads/master /.git-hashref +# ADD https://api.github.com/repos/christiaanjs/treeflow/git/refs/heads/master /.git-hashref RUN git clone --depth 1 https://github.com/christiaanjs/treeflow.git /treeflow RUN cd /treeflow && git checkout e3414dcc9e764d06abc3e19c1d0f55110499e2ea && /opt/conda/envs/bito/bin/pip install . RUN . /opt/conda/etc/profile.d/conda.sh && conda activate bito && treeflow_benchmark --help From 6fb4d24abe7dc6bccf08d93b75b16a5f31f5797e Mon Sep 17 00:00:00 2001 From: jgallowa07 Date: Wed, 26 Oct 2022 06:29:14 -0700 Subject: [PATCH 18/19] patch #2 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ddbc045..9c49501 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,7 +73,7 @@ RUN pip install phylostan==1.0.5 && phylostan --help RUN pip install tqdm # workaround to bust cache if the master branch has been updated ... -# ADD https://api.github.com/repos/christiaanjs/treeflow/git/refs/heads/master /.git-hashref +ADD https://api.github.com/repos/christiaanjs/treeflow/git/refs/heads/master /.git-hashref RUN git clone --depth 1 https://github.com/christiaanjs/treeflow.git /treeflow RUN cd /treeflow && git checkout e3414dcc9e764d06abc3e19c1d0f55110499e2ea && /opt/conda/envs/bito/bin/pip install . RUN . /opt/conda/etc/profile.d/conda.sh && conda activate bito && treeflow_benchmark --help From 331208e6c4714f96b070217d1b086ed03cded62b Mon Sep 17 00:00:00 2001 From: jgallowa07 Date: Wed, 26 Oct 2022 12:11:42 -0700 Subject: [PATCH 19/19] patch #3 --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9c49501..6e0b39f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,8 +73,8 @@ RUN pip install phylostan==1.0.5 && phylostan --help RUN pip install tqdm # workaround to bust cache if the master branch has been updated ... -ADD https://api.github.com/repos/christiaanjs/treeflow/git/refs/heads/master /.git-hashref -RUN git clone --depth 1 https://github.com/christiaanjs/treeflow.git /treeflow +# ADD https://api.github.com/repos/christiaanjs/treeflow/git/refs/heads/master /.git-hashref +RUN git clone --depth 10 https://github.com/christiaanjs/treeflow.git /treeflow RUN cd /treeflow && git checkout e3414dcc9e764d06abc3e19c1d0f55110499e2ea && /opt/conda/envs/bito/bin/pip install . RUN . /opt/conda/etc/profile.d/conda.sh && conda activate bito && treeflow_benchmark --help