diff --git a/README.md b/README.md index 8e7291e..69f179a 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,21 @@ The `source` option may be used to specify the URL to download the scaffold files from; the default source is drupal.org. The literal string `{version}` in the `source` option is replaced with the current version of Drupal core being updated prior to download. +You can also define `source` as an array to have fallbacks in case of +any HTTP issues. + +```json +{ + "extra": { + "drupal-scaffold": { + "source": [ + "https://cgit.drupalcode.org/drupal/plain/{path}?h={version}", + "https://raw.githubusercontent.com/drupal/drupal/{version}/{path}" + ] + } + } +} +``` With the `drupal-scaffold` option `excludes`, you can provide additional paths that should not be copied or overwritten. The plugin provides no excludes by diff --git a/src/FileFetcher.php b/src/FileFetcher.php index 59b9227..ac38390 100644 --- a/src/FileFetcher.php +++ b/src/FileFetcher.php @@ -49,13 +49,19 @@ class FileFetcher { */ protected $fs; + /** + * @var array + * + * A list of potential errors. + */ + protected $errors = []; + /** * Constructs this FileFetcher object. */ - public function __construct(RemoteFilesystem $remoteFilesystem, $source, IOInterface $io, $progress = TRUE) { + public function __construct(RemoteFilesystem $remoteFilesystem, IOInterface $io, $progress = TRUE) { $this->remoteFilesystem = $remoteFilesystem; $this->io = $io; - $this->source = $source; $this->fs = new Filesystem(); $this->progress = $progress; } @@ -64,23 +70,39 @@ public function __construct(RemoteFilesystem $remoteFilesystem, $source, IOInter * Downloads all required files and writes it to the file system. */ public function fetch($version, $destination, $override) { + $errors = []; + foreach ($this->filenames as $sourceFilename => $filename) { $target = "$destination/$filename"; if ($override || !file_exists($target)) { $url = $this->getUri($sourceFilename, $version); $this->fs->ensureDirectoryExists($destination . '/' . dirname($filename)); + if ($this->progress) { $this->io->writeError(" - $filename ($url): ", FALSE); - $this->remoteFilesystem->copy($url, $url, $target, $this->progress); - // Used to put a new line because the remote file system does not put - // one. + try { + $this->remoteFilesystem->copy($url, $url, $target, $this->progress); + } catch(\Exception $e) { + $errors[] = $url; + } + // New line because the remoteFilesystem does not put one. $this->io->writeError(''); } else { - $this->remoteFilesystem->copy($url, $url, $target, $this->progress); + try { + $this->remoteFilesystem->copy($url, $url, $target, $this->progress); + } catch(\Exception $e) { + $errors[] = $url; + } } } } + + if ($errors) { + $this->addError('Failed to download: ' . "\r\n" . implode("\r\n", $errors)); + return FALSE; + } + return TRUE; } /** @@ -90,6 +112,27 @@ public function setFilenames(array $filenames) { $this->filenames = $filenames; } + /** + * Set source. + */ + public function setSource($source) { + $this->source = $source; + } + + /** + * Set error. + */ + public function addError($error) { + $this->errors[] = $error; + } + + /** + * Get errors. + */ + public function getErrors() { + return $this->errors; + } + /** * Replace filename and version in the source pattern with their values. */ diff --git a/src/Handler.php b/src/Handler.php index 23cb9d9..51f1233 100644 --- a/src/Handler.php +++ b/src/Handler.php @@ -149,6 +149,7 @@ public function downloadScaffold() { // Collect options, excludes and settings files. $options = $this->getOptions(); $files = array_diff($this->getIncludes(), $this->getExcludes()); + $files = array_combine($files, $files); // Call any pre-scaffold scripts that may be defined. $dispatcher = new EventDispatcher($this->composer, $this->io); @@ -158,12 +159,39 @@ public function downloadScaffold() { $remoteFs = new RemoteFilesystem($this->io); - $fetcher = new PrestissimoFileFetcher($remoteFs, $options['source'], $this->io, $this->progress, $this->composer->getConfig()); - $fetcher->setFilenames(array_combine($files, $files)); - $fetcher->fetch($version, $webroot, TRUE); + $fetcher = new PrestissimoFileFetcher($remoteFs, $this->io, $this->progress, $this->composer->getConfig()); + $sources = (array) $options['source']; + $all_succeeded = FALSE; - $fetcher->setFilenames($this->getInitial()); - $fetcher->fetch($version, $webroot, FALSE); + do { + $source = current($sources); + + $fetcher->setSource($source); + + $fetcher->setFilenames($files); + if ($fetcher->fetch($version, $webroot, TRUE)) { + $fetcher->setFilenames($this->getInitial()); + if ($fetcher->fetch($version, $webroot, FALSE)) { + $all_succeeded = TRUE; + break; + } + } + + // If here, it means that the fetch for this source has failed. + $next_source = next($sources); + + $this->io->writeError(''); + $this->io->writeError(" - Has failed with the " . (!$next_source ? 'last ' : '') . "source: $source", TRUE); + if ($next_source) { + $this->io->writeError(" - Now trying with the source: $next_source", TRUE); + } + $this->io->writeError(''); + + } while($next_source); + + if (!$all_succeeded) { + throw new \Exception(implode("\r\n\r\n", $fetcher->getErrors())); + } // Call post-scaffold scripts. $dispatcher->dispatch(self::POST_DRUPAL_SCAFFOLD_CMD); @@ -343,8 +371,10 @@ protected function getOptions() { 'excludes' => [], 'includes' => [], 'initial' => [], - 'source' => 'https://cgit.drupalcode.org/drupal/plain/{path}?h={version}', - // Github: https://raw.githubusercontent.com/drupal/drupal/{version}/{path} + 'source' => [ + 'https://cgit.drupalcode.org/drupal/plain/{path}?h={version}', + 'https://raw.githubusercontent.com/drupal/drupal/{version}/{path}' + ], ]; return $options; } diff --git a/src/PrestissimoFileFetcher.php b/src/PrestissimoFileFetcher.php index 30b7c23..25dccd7 100644 --- a/src/PrestissimoFileFetcher.php +++ b/src/PrestissimoFileFetcher.php @@ -22,8 +22,8 @@ class PrestissimoFileFetcher extends FileFetcher { /** * Constructs this PrestissimoFileFetcher object. */ - public function __construct(RemoteFilesystem $remoteFilesystem, $source, IOInterface $io, $progress = TRUE, Config $config) { - parent::__construct($remoteFilesystem, $source, $io, $progress); + public function __construct(RemoteFilesystem $remoteFilesystem, IOInterface $io, $progress = TRUE, Config $config) { + parent::__construct($remoteFilesystem, $io, $progress); $this->config = $config; } @@ -32,10 +32,9 @@ public function __construct(RemoteFilesystem $remoteFilesystem, $source, IOInter */ public function fetch($version, $destination, $override) { if (class_exists(CurlMulti::class)) { - $this->fetchWithPrestissimo($version, $destination, $override); - return; + return $this->fetchWithPrestissimo($version, $destination, $override); } - parent::fetch($version, $destination, $override); + return parent::fetch($version, $destination, $override); } /** @@ -57,7 +56,7 @@ protected function fetchWithPrestissimo($version, $destination, $override) { $errors = []; $totalCnt = count($requests); if ($totalCnt == 0) { - return; + return TRUE; } $multi = new CurlMulti(); @@ -70,6 +69,9 @@ protected function fetchWithPrestissimo($version, $destination, $override) { $failureCnt += $result['failureCnt']; if (isset($result['errors'])) { $errors += $result['errors']; + foreach ($result['errors'] as $url => $error) { + $this->io->writeError(" - Downloading $successCnt/$totalCnt: $url (failed)", TRUE); + } } if ($this->progress) { foreach ($result['urls'] as $url) { @@ -78,10 +80,20 @@ protected function fetchWithPrestissimo($version, $destination, $override) { } } while ($multi->remain()); - $urls = array_keys($errors); - if ($urls) { - throw new \Exception('Failed to download ' . implode(", ", $urls)); + if ($errors) { + $this->addError('Failed to download: ' . "\r\n" . implode("\r\n", array_keys($errors))); + $errors_extra = []; + foreach($errors as $error) { + if ($error !== "0: " && !isset($errors_extra[$error])) { + $errors_extra[$error] = $error; + } + } + if ($errors_extra) { + $this->addError(implode("\r\n", $errors_extra)); + } + return FALSE; } + return TRUE; } } diff --git a/tests/FetcherTest.php b/tests/FetcherTest.php index ea822f9..df2a973 100644 --- a/tests/FetcherTest.php +++ b/tests/FetcherTest.php @@ -57,7 +57,8 @@ protected function ensureDirectoryExistsAndClear($directory) { } public function testFetch() { - $fetcher = new FileFetcher(new RemoteFilesystem(new NullIO()), 'https://cgit.drupalcode.org/drupal/plain/{path}?h={version}', new NullIO()); + $fetcher = new FileFetcher(new RemoteFilesystem(new NullIO()), new NullIO()); + $fetcher->setSource('https://cgit.drupalcode.org/drupal/plain/{path}?h={version}'); $fetcher->setFilenames([ '.htaccess' => '.htaccess', 'sites/default/default.settings.php' => 'sites/default/default.settings.php', @@ -68,7 +69,8 @@ public function testFetch() { } public function testInitialFetch() { - $fetcher = new FileFetcher(new RemoteFilesystem(new NullIO()), 'https://cgit.drupalcode.org/drupal/plain/{path}?h={version}', new NullIO()); + $fetcher = new FileFetcher(new RemoteFilesystem(new NullIO()), new NullIO()); + $fetcher->setSource('https://cgit.drupalcode.org/drupal/plain/{path}?h={version}'); $fetcher->setFilenames([ 'sites/default/default.settings.php' => 'sites/default/settings.php', ]);