Remove prebuilt nupkgs from the tarball. (#1348)

- Keep a list of packages that are detected as prebuilts even though they aren't used in the tarball build.
- Delete these packages after doing the normal filtering.
- Enforce zero prebuilts in build-source-tarball, with a whitelist in case we need to regress on prebuilts.

* Add an option to skip zero prebuilt enforcement.

* Don't fail on missing prebuilts

Author: crummel

build-source-tarball.sh

@@ -18,6 +18,7 @@ shift
 SKIP_BUILD=0
 INCLUDE_LEAK_DETECTION=0
 MINIMIZE_DISK_USAGE=0
+SKIP_PREBUILT_ENFORCEMENT=0
 export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 
 while :; do
@@ -36,6 +37,9 @@ while :; do
         --minimize-disk-usage)
             MINIMIZE_DISK_USAGE=1
             ;;
+        --skip-prebuilt-check)
+            SKIP_PREBUILT_ENFORCEMENT=1
+            ;;
         --)
             shift
             echo "Detected '--': passing remaining parameters '$@' as build.sh arguments."
@@ -173,6 +177,14 @@ find $TARBALL_ROOT/src \( -type f \( \
     -iname *.zip -o \
     -iname *.nupkg \) \) -exec rm {} \;
 
+if [ $MINIMIZE_DISK_USAGE -eq 1 ]; then
+    pushd "$TARBALL_ROOT/src"
+    echo 'Removing unneeded files to trim tarball...'
+    # we don't build CoreCLR tests right now and they have a lot of them - ~380MB
+    rm -rf coreclr.*/tests
+    popd
+fi
+
 echo 'Copying sourcelink metadata to tarball...'
 pushd $SCRIPT_ROOT
 for srcDir in `find bin/src -name '.git' -type d`; do
@@ -289,6 +301,29 @@ do
     fi
 done
 
+echo 'Removing known extra packages from tarball prebuilts...'
+while IFS= read -r packagePattern
+do
+    if [[ "$packagePattern" =~ ^# ]]; then
+        continue
+    fi
+    rm -f $TARBALL_ROOT/packages/prebuilt/$packagePattern
+done < $SCRIPT_ROOT/support/additional-prebuilts-to-delete.txt
+
+if [ $SKIP_PREBUILT_ENFORCEMENT -ne 1 ]; then
+  echo 'Checking for extra prebuilts...'
+  for package in `ls -A $TARBALL_ROOT/packages/prebuilt`
+  do
+      if grep -q "$package" $SCRIPT_ROOT/support/allowed-prebuilts.txt; then
+          echo "Allowing prebuilt $package"
+      else
+          echo "ERROR: $package is not in the allowed prebuilts list ($SCRIPT_ROOT/support/allowed-prebuilts.txt)"
+          echo "Either remove this prebuilt, add it to the known extras list ($SCRIPT_ROOT/support/additional-prebuilts-to-delete.txt) or add it to the allowed prebuilts list."
+          exit 1
+      fi
+  done
+fi
+
 echo 'Removing source-built, previously source-built packages and reference packages from il pkg src...'
 OLDIFS=$IFS
 

support/additional-prebuilts-to-delete.txt

@@ -0,0 +1,33 @@
+# this is a list of packages that end up in the tarball's prebuilts folder but are not actually needed.
+# this is typically because part of the build are turned off for DotNetBuildOffline but not DotNetBuildFromSource.
+microsoft.dotnet.arcade.sdk.*.nupkg
+microsoft.diasymreader.pdb2pdb.*.nupkg
+microsoft.dotnet.signtool.*.nupkg
+sn.1.0.0.nupkg
+vswhere.2.6.7.nupkg
+system.text.encoding.codepages.4.3.0.nupkg
+system.text.encoding.codepages.4.5.1.nupkg
+system.threading.tasks.dataflow.4.5.24.nupkg
+runtime.*.runtime.native.system.security.cryptography.apple.*.nupkg
+runtime.*.runtime.native.system.security.cryptography.openssl.*.nupkg
+system.text.json.1.0.0.nupkg
+system.xml.xpath.4.3.0.nupkg
+microsoft.codeanalysis.csharp.2.9.0.nupkg
+system.xml.xpath.xdocument.4.3.0.nupkg
+microsoft.codeanalysis.common.2.9.0.nupkg
+microsoft.codeanalysis.analyzers.2.6.1.nupkg
+system.valuetuple.4.3.0.nupkg
+microsoft.dotnet.genapi.*.nupkg
+system.runtime.compilerservices.unsafe.4.5.0.nupkg
+runtime.linux-x64.microsoft.netcore.dotnetapphost.*.nupkg
+runtime.linux-x64.microsoft.netcore.dotnethostresolver.*.nupkg
+runtime.linux-x64.microsoft.netcore.dotnethostpolicy.*.nupkg
+runtime.linux-x64.microsoft.netcore.dotnethost.*.nupkg
+microsoft.netcore.app.host.linux-x64.3.0.0.nupkg
+system.composition.runtime.1.0.31.nupkg
+system.composition.hosting.1.0.31.nupkg
+system.composition.typedparts.1.0.31.nupkg
+system.composition.convention.1.0.31.nupkg
+system.composition.attributedmodel.1.0.31.nupkg
+microsoft.visualstudio.setup.configuration.interop.1.16.30.nupkg
+microsoft.net.compilers.toolset.*.nupkg

support/allowed-prebuilts.txt

@@ -0,0 +1 @@
+