[AssetMapper] Fix: also download files referenced by url() in CSS

Author: weaverryan

Command/ImportMapInstallCommand.php

@@ -64,7 +64,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
         }
 
         $io->success(sprintf(
-            'Downloaded %d asset%s into %s.',
+            'Downloaded %d package%s into %s.',
             \count($downloadedPackages),
             1 === \count($downloadedPackages) ? '' : 's',
             str_replace($this->projectDir.'/', '', $this->packageDownloader->getVendorDir()),

ImportMap/RemotePackageDownloader.php

@@ -52,6 +52,7 @@ public function downloadPackages(callable $progressCallback = null): array
                 isset($installed[$entry->importName])
                 && $installed[$entry->importName]['version'] === $entry->version
                 && $this->remotePackageStorage->isDownloaded($entry)
+                && $this->areAllExtraFilesDownloaded($entry, $installed[$entry->importName]['extraFiles'])
             ) {
                 $newInstalled[$entry->importName] = $installed[$entry->importName];
                 continue;
@@ -72,9 +73,14 @@ public function downloadPackages(callable $progressCallback = null): array
             }
 
             $this->remotePackageStorage->save($entry, $contents[$package]['content']);
+            foreach ($contents[$package]['extraFiles'] as $extraFilename => $extraFileContents) {
+                $this->remotePackageStorage->saveExtraFile($entry, $extraFilename, $extraFileContents);
+            }
+
             $newInstalled[$package] = [
                 'version' => $entry->version,
                 'dependencies' => $contents[$package]['dependencies'] ?? [],
+                'extraFiles' => array_keys($contents[$package]['extraFiles']),
             ];
 
             $downloadedPackages[] = $package;
@@ -109,7 +115,7 @@ public function getVendorDir(): string
     }
 
     /**
-     * @return array<string, array{path: string, version: string, dependencies: array<string, string>}>
+     * @return array<string, array{path: string, version: string, dependencies: array<string, string>, extraFiles: array<string, string>}>
      */
     private function loadInstalled(): array
     {
@@ -128,6 +134,10 @@ private function loadInstalled(): array
             if (!isset($data['dependencies'])) {
                 throw new \LogicException(sprintf('The package "%s" is missing its dependencies.', $package));
             }
+
+            if (!isset($data['extraFiles'])) {
+                $installed[$package]['extraFiles'] = [];
+            }
         }
 
         return $this->installed = $installed;
@@ -138,4 +148,15 @@ private function saveInstalled(array $installed): void
         $this->installed = $installed;
         file_put_contents($this->remotePackageStorage->getStorageDir().'/installed.php', sprintf('<?php return %s;', var_export($installed, true)));
     }
+
+    private function areAllExtraFilesDownloaded(ImportMapEntry $entry, array $extraFilenames): bool
+    {
+        foreach ($extraFilenames as $extraFilename) {
+            if (!$this->remotePackageStorage->isExtraFileDownloaded($entry, $extraFilename)) {
+                return false;
+            }
+        }
+
+        return true;
+    }
 }

ImportMap/RemotePackageStorage.php

@@ -34,6 +34,15 @@ public function isDownloaded(ImportMapEntry $entry): bool
         return is_file($this->getDownloadPath($entry->packageModuleSpecifier, $entry->type));
     }
 
+    public function isExtraFileDownloaded(ImportMapEntry $entry, string $extraFilename): bool
+    {
+        if (!$entry->isRemotePackage()) {
+            throw new \InvalidArgumentException(sprintf('The entry "%s" is not a remote package.', $entry->importName));
+        }
+
+        return is_file($this->getExtraFileDownloadPath($entry, $extraFilename));
+    }
+
     public function save(ImportMapEntry $entry, string $contents): void
     {
         if (!$entry->isRemotePackage()) {
@@ -46,6 +55,18 @@ public function save(ImportMapEntry $entry, string $contents): void
         file_put_contents($vendorPath, $contents);
     }
 
+    public function saveExtraFile(ImportMapEntry $entry, string $extraFilename, string $contents): void
+    {
+        if (!$entry->isRemotePackage()) {
+            throw new \InvalidArgumentException(sprintf('The entry "%s" is not a remote package.', $entry->importName));
+        }
+
+        $vendorPath = $this->getExtraFileDownloadPath($entry, $extraFilename);
+
+        @mkdir(\dirname($vendorPath), 0777, true);
+        file_put_contents($vendorPath, $contents);
+    }
+
     /**
      * The local file path where a downloaded package should be stored.
      */
@@ -68,4 +89,9 @@ public function getDownloadPath(string $packageModuleSpecifier, ImportMapType $i
 
         return $this->vendorDir.'/'.$filename;
     }
+
+    private function getExtraFileDownloadPath(ImportMapEntry $entry, string $extraFilename): string
+    {
+        return $this->vendorDir.'/'.$entry->getPackageName().'/'.ltrim($extraFilename, '/');
+    }
 }

ImportMap/Resolver/JsDelivrEsmResolver.php

@@ -11,10 +11,12 @@
 
 namespace Symfony\Component\AssetMapper\ImportMap\Resolver;
 
+use Symfony\Component\AssetMapper\Compiler\CssAssetUrlCompiler;
 use Symfony\Component\AssetMapper\Exception\RuntimeException;
 use Symfony\Component\AssetMapper\ImportMap\ImportMapEntry;
 use Symfony\Component\AssetMapper\ImportMap\ImportMapType;
 use Symfony\Component\AssetMapper\ImportMap\PackageRequireOptions;
+use Symfony\Component\Filesystem\Path;
 use Symfony\Component\HttpClient\HttpClient;
 use Symfony\Contracts\HttpClient\Exception\HttpExceptionInterface;
 use Symfony\Contracts\HttpClient\HttpClientInterface;
@@ -157,12 +159,11 @@ public function resolvePackages(array $packagesToRequire): array
     /**
      * @param ImportMapEntry[] $importMapEntries
      *
-     * @return array<string, array{content: string, dependencies: string[]}>
+     * @return array<string, array{content: string, dependencies: string[], extraFiles: array<string, string>}>
      */
     public function downloadPackages(array $importMapEntries, callable $progressCallback = null): array
     {
         $responses = [];
-
         foreach ($importMapEntries as $package => $entry) {
             if (!$entry->isRemotePackage()) {
                 throw new \InvalidArgumentException(sprintf('The entry "%s" is not a remote package.', $entry->importName));
@@ -171,12 +172,13 @@ public function downloadPackages(array $importMapEntries, callable $progressCall
             $pattern = ImportMapType::CSS === $entry->type ? self::URL_PATTERN_DIST_CSS : self::URL_PATTERN_DIST;
             $url = sprintf($pattern, $entry->getPackageName(), $entry->version, $entry->getPackagePathString());
 
-            $responses[$package] = $this->httpClient->request('GET', $url);
+            $responses[$package] = [$this->httpClient->request('GET', $url), $entry];
         }
 
         $errors = [];
         $contents = [];
-        foreach ($responses as $package => $response) {
+        $extraFileResponses = [];
+        foreach ($responses as $package => [$response, $entry]) {
             if (200 !== $response->getStatusCode()) {
                 $errors[] = [$package, $response];
                 continue;
@@ -187,10 +189,21 @@ public function downloadPackages(array $importMapEntries, callable $progressCall
             }
 
             $dependencies = [];
+            $extraFiles = [];
+            /* @var ImportMapEntry $entry */
             $contents[$package] = [
-                'content' => $this->makeImportsBare($response->getContent(), $dependencies),
+                'content' => $this->makeImportsBare($response->getContent(), $dependencies, $extraFiles, $entry->type, $entry->getPackagePathString()),
                 'dependencies' => $dependencies,
+                'extraFiles' => [],
             ];
+
+            if (0 !== \count($extraFiles)) {
+                $extraFileResponses[$package] = [];
+                foreach ($extraFiles as $extraFile) {
+                    $extraFileResponses[$package][] = [$this->httpClient->request('GET', sprintf(self::URL_PATTERN_DIST_CSS, $entry->getPackageName(), $entry->version, $extraFile)), $extraFile, $entry->getPackageName(), $entry->version];
+                }
+            }
+
             if ($progressCallback) {
                 $progressCallback($package, 'finished', $response, \count($responses));
             }
@@ -205,6 +218,47 @@ public function downloadPackages(array $importMapEntries, callable $progressCall
             throw new RuntimeException(sprintf('Error %d downloading packages from jsDelivr for "%s". Check your package names. Response: ', $response->getStatusCode(), $packages).$response->getContent(false), 0, $e);
         }
 
+        $extraFileErrors = [];
+        download_extra_files:
+        $packageFileResponses = $extraFileResponses;
+        $extraFileResponses = [];
+        foreach ($packageFileResponses as $package => $responses) {
+            foreach ($responses as [$response, $extraFile, $packageName, $version]) {
+                if (200 !== $response->getStatusCode()) {
+                    $extraFileErrors[] = [$package, $response];
+                    continue;
+                }
+
+                $extraFiles = [];
+
+                $content = $response->getContent();
+                if (str_ends_with($extraFile, '.css')) {
+                    $content = $this->makeImportsBare($content, $dependencies, $extraFiles, ImportMapType::CSS, $extraFile);
+                }
+                $contents[$package]['extraFiles'][$extraFile] = $content;
+
+                if (0 !== \count($extraFiles)) {
+                    $extraFileResponses[$package] = [];
+                    foreach ($extraFiles as $newExtraFile) {
+                        $extraFileResponses[$package][] = [$this->httpClient->request('GET', sprintf(self::URL_PATTERN_DIST_CSS, $packageName, $version, $newExtraFile)), $newExtraFile, $packageName, $version];
+                    }
+                }
+            }
+        }
+
+        if ($extraFileResponses) {
+            goto download_extra_files;
+        }
+
+        try {
+            ($extraFileErrors[0][1] ?? null)?->getHeaders();
+        } catch (HttpExceptionInterface $e) {
+            $response = $e->getResponse();
+            $packages = implode('", "', array_column($extraFileErrors, 0));
+
+            throw new RuntimeException(sprintf('Error %d downloading extra imported files from jsDelivr for "%s". Response: ', $response->getStatusCode(), $packages).$response->getContent(false), 0, $e);
+        }
+
         return $contents;
     }
 
@@ -237,20 +291,37 @@ private function fetchPackageRequirementsFromImports(string $content): array
      *
      * Replaces those with normal import "package/name" statements.
      */
-    private function makeImportsBare(string $content, array &$dependencies): string
+    private function makeImportsBare(string $content, array &$dependencies, array &$extraFiles, ImportMapType $type, string $sourceFilePath): string
     {
-        $content = preg_replace_callback(self::IMPORT_REGEX, function ($matches) use (&$dependencies) {
-            $packageName = $matches[2].$matches[4]; // add the path if any
-            $dependencies[] = $packageName;
-
-            // replace the "/npm/package@version/+esm" with "package@version"
-            return str_replace($matches[1], sprintf('"%s"', $packageName), $matches[0]);
-        }, $content);
-
-        // source maps are not also downloaded - so remove the sourceMappingURL
-        // remove the final one only (in case sourceMappingURL is used in the code)
-        if (false !== $lastPos = strrpos($content, '//# sourceMappingURL=')) {
-            $content = substr($content, 0, $lastPos).preg_replace('{//# sourceMappingURL=.*$}m', '', substr($content, $lastPos));
+        if (ImportMapType::JS === $type) {
+            $content = preg_replace_callback(self::IMPORT_REGEX, function ($matches) use (&$dependencies) {
+                $packageName = $matches[2].$matches[4]; // add the path if any
+                $dependencies[] = $packageName;
+
+                // replace the "/npm/package@version/+esm" with "package@version"
+                return str_replace($matches[1], sprintf('"%s"', $packageName), $matches[0]);
+            }, $content);
+
+            // source maps are not also downloaded - so remove the sourceMappingURL
+            // remove the final one only (in case sourceMappingURL is used in the code)
+            if (false !== $lastPos = strrpos($content, '//# sourceMappingURL=')) {
+                $content = substr($content, 0, $lastPos).preg_replace('{//# sourceMappingURL=.*$}m', '', substr($content, $lastPos));
+            }
+
+            return $content;
+        }
+
+        preg_match_all(CssAssetUrlCompiler::ASSET_URL_PATTERN, $content, $matches);
+        foreach ($matches[1] as $path) {
+            if (str_starts_with($path, 'data:')) {
+                continue;
+            }
+
+            if (str_starts_with($path, 'http://') || str_starts_with($path, 'https://')) {
+                continue;
+            }
+
+            $extraFiles[] = Path::join(\dirname($sourceFilePath), $path);
         }
 
         return preg_replace('{/\*# sourceMappingURL=[^ ]*+ \*/}', '', $content);

ImportMap/Resolver/PackageResolverInterface.php

@@ -37,7 +37,7 @@ public function resolvePackages(array $packagesToRequire): array;
      *
      * @param array<string, ImportMapEntry> $importMapEntries
      *
-     * @return array<string, array{content: string, dependencies: string[]}>
+     * @return array<string, array{content: string, dependencies: string[], extraFiles: array<string, string>}>
      */
     public function downloadPackages(array $importMapEntries, callable $progressCallback = null): array;
 }