From 8a36362591cce5fc149c7307d1639a1f8c8547ad Mon Sep 17 00:00:00 2001 From: Chuan Zhang Date: Mon, 16 Sep 2019 11:09:07 -0400 Subject: [PATCH 1/6] Add a retry to adjust for network failures when downloading from sourceforge --- packaging/build_from_source.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging/build_from_source.sh b/packaging/build_from_source.sh index 40e2a8e14b..f85d02d60c 100755 --- a/packaging/build_from_source.sh +++ b/packaging/build_from_source.sh @@ -10,10 +10,10 @@ rm -rf /tmp/torchaudio-deps mkdir /tmp/torchaudio-deps pushd /tmp/torchaudio-deps -curl -L -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" -curl -L -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" -curl -L -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" -curl -L -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" +curl -L --retry 3 -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" +curl -L --retry 3 -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" +curl -L --retry 3 -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" +curl -L --retry 3 -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" # unpack the dependencies tar xfp sox-14.4.2.tar.bz2 From 1776e3a08a1a9b7e9ef0f2c5c2305ebdbd13907c Mon Sep 17 00:00:00 2001 From: Chuan Zhang Date: Mon, 16 Sep 2019 13:51:43 -0400 Subject: [PATCH 2/6] try to fix packaging by adding longer connection timeout --- packaging/build_from_source.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packaging/build_from_source.sh b/packaging/build_from_source.sh index f85d02d60c..26215473be 100755 --- a/packaging/build_from_source.sh +++ b/packaging/build_from_source.sh @@ -10,10 +10,10 @@ rm -rf /tmp/torchaudio-deps mkdir /tmp/torchaudio-deps pushd /tmp/torchaudio-deps -curl -L --retry 3 -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" -curl -L --retry 3 -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" -curl -L --retry 3 -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" -curl -L --retry 3 -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" +curl -L --connect-timeout 120 -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" +curl -L --connect-timeout 120 -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" +curl -L --connect-timeout 120 -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" +curl -L --connect-timeout 120 -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" # unpack the dependencies tar xfp sox-14.4.2.tar.bz2 From 821c3e2ddbf0df205460332798ee5a754096a95c Mon Sep 17 00:00:00 2001 From: Chuan Zhang Date: Mon, 16 Sep 2019 14:51:44 -0400 Subject: [PATCH 3/6] trying a faster connection timeout --- packaging/build_from_source.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packaging/build_from_source.sh b/packaging/build_from_source.sh index 26215473be..b14e96e3c0 100755 --- a/packaging/build_from_source.sh +++ b/packaging/build_from_source.sh @@ -10,10 +10,12 @@ rm -rf /tmp/torchaudio-deps mkdir /tmp/torchaudio-deps pushd /tmp/torchaudio-deps -curl -L --connect-timeout 120 -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" -curl -L --connect-timeout 120 -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" -curl -L --connect-timeout 120 -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" -curl -L --connect-timeout 120 -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" +CURL_OPTS=" --retry 5 --connect-timeout 15 " + +curl -L $CURL_OPTS -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" +curl -L $CURL_OPTS -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" +curl -L $CURL_OPTS -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" +curl -L $CURL_OPTS -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" # unpack the dependencies tar xfp sox-14.4.2.tar.bz2 From e8a1ad30d3540d363a749117c6f22a194f61f65d Mon Sep 17 00:00:00 2001 From: Chuan Zhang Date: Mon, 16 Sep 2019 14:58:20 -0400 Subject: [PATCH 4/6] trying more retries with faster time to connect --- packaging/build_from_source.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/build_from_source.sh b/packaging/build_from_source.sh index b14e96e3c0..1f4c9ff8c7 100755 --- a/packaging/build_from_source.sh +++ b/packaging/build_from_source.sh @@ -10,7 +10,7 @@ rm -rf /tmp/torchaudio-deps mkdir /tmp/torchaudio-deps pushd /tmp/torchaudio-deps -CURL_OPTS=" --retry 5 --connect-timeout 15 " +CURL_OPTS=" --retry 8 --connect-timeout 8 " curl -L $CURL_OPTS -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" curl -L $CURL_OPTS -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" From c101d392298912fef56a2b6444a2bff17e29f6bd Mon Sep 17 00:00:00 2001 From: Chuan Zhang Date: Tue, 17 Sep 2019 08:03:23 -0400 Subject: [PATCH 5/6] Twiddling with CURL Options --- packaging/build_from_source.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packaging/build_from_source.sh b/packaging/build_from_source.sh index 1f4c9ff8c7..423b2a6691 100755 --- a/packaging/build_from_source.sh +++ b/packaging/build_from_source.sh @@ -10,10 +10,14 @@ rm -rf /tmp/torchaudio-deps mkdir /tmp/torchaudio-deps pushd /tmp/torchaudio-deps -CURL_OPTS=" --retry 8 --connect-timeout 8 " -curl -L $CURL_OPTS -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Fsox%2Ffiles%2Fsox%2F14.4.2%2F&ts=1416316415&use_mirror=heanet" -curl -L $CURL_OPTS -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz?r=http%3A%2F%2Fsourceforge.net%2Fprojects%2Flame%2Ffiles%2Flame%2F3.99%2F&ts=1416316457&use_mirror=kent" +# Curl Settings +# 3 minutes is the absolute max for the curl command +# Retry up to 10 times, wait to connect at most 5s per time +CURL_OPTS=" --retry 10 --connect-timeout 5 --max-time 180 " + +curl -L $CURL_OPTS -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2" +curl -L $CURL_OPTS -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz" curl -L $CURL_OPTS -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" curl -L $CURL_OPTS -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" From 3fef1e24f1e05b84d1965d89ff126dc7602ddaa6 Mon Sep 17 00:00:00 2001 From: Chuan Zhang Date: Tue, 17 Sep 2019 17:42:52 -0400 Subject: [PATCH 6/6] small tweaks to CURL_OPTS --- packaging/build_from_source.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packaging/build_from_source.sh b/packaging/build_from_source.sh index 423b2a6691..f4a3e7f875 100755 --- a/packaging/build_from_source.sh +++ b/packaging/build_from_source.sh @@ -14,12 +14,12 @@ pushd /tmp/torchaudio-deps # Curl Settings # 3 minutes is the absolute max for the curl command # Retry up to 10 times, wait to connect at most 5s per time -CURL_OPTS=" --retry 10 --connect-timeout 5 --max-time 180 " +CURL_OPTS="-L --retry 10 --connect-timeout 5 --max-time 180" -curl -L $CURL_OPTS -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2" -curl -L $CURL_OPTS -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz" -curl -L $CURL_OPTS -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" -curl -L $CURL_OPTS -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" +curl $CURL_OPTS -o sox-14.4.2.tar.bz2 "http://downloads.sourceforge.net/project/sox/sox/14.4.2/sox-14.4.2.tar.bz2" +curl $CURL_OPTS -o lame-3.99.5.tar.gz "http://downloads.sourceforge.net/project/lame/lame/3.99/lame-3.99.5.tar.gz" +curl $CURL_OPTS -o flac-1.3.2.tar.xz "https://superb-dca2.dl.sourceforge.net/project/flac/flac-src/flac-1.3.2.tar.xz" +curl $CURL_OPTS -o libmad-0.15.1b.tar.gz "https://downloads.sourceforge.net/project/mad/libmad/0.15.1b/libmad-0.15.1b.tar.gz" # unpack the dependencies tar xfp sox-14.4.2.tar.bz2