Merge branch '7.0' into 7.1

xabbuh · xabbuh · commit bd95975f7304 · 2024-04-13T08:11:48.000+02:00
* 7.0: (39 commits)
  fix merge
  add missing return type-hints
  fix merge
  explicitly mark nullable parameters as nullable
  fix low deps tests
  [HttpKernel] Fix datacollector caster for reference object property
  [Serializer] Fixing PHP warning in the ObjectNormalizer with MaxDepth enabled
  bug #51578 [Cache] always select database for persistent redis connections
  [Security] Validate that CSRF token in form login is string similar to username/password
  [Serializer] Use explicit nullable type
  [validator] validated Dutch translation
  Improve dutch translations
  initialize the current time with midnight before modifying the date
  [Translation] Skip state=needs-translation entries only when source == target
  [HttpKernel] Ensure controllers are not lazy
  [Validator] Fill in trans-unit id 113: This URL does not contain a TLD.
  [Validator] added missing Polish translation for unit 113
  [Validator] add missing lv translation
  fix tests
  [HttpClient] Let curl handle transfer encoding
  ...
diff --git a/Crawler.php b/Crawler.php
@@ -1061,12 +1061,30 @@ protected function sibling(\DOMNode $node, string $siblingDir = 'nextSibling'):
 
     private function parseHtml5(string $htmlContent, string $charset = 'UTF-8'): \DOMDocument
     {
-        return $this->html5Parser->parse($this->convertToHtmlEntities($htmlContent, $charset));
+        if (!$this->supportsEncoding($charset)) {
+            $htmlContent = $this->convertToHtmlEntities($htmlContent, $charset);
+            $charset = 'UTF-8';
+        }
+
+        return $this->html5Parser->parse($htmlContent, ['encoding' => $charset]);
+    }
+
+    private function supportsEncoding(string $encoding): bool
+    {
+        try {
+            return '' === @mb_convert_encoding('', $encoding, 'UTF-8');
+        } catch (\Throwable $e) {
+            return false;
+        }
     }
 
     private function parseXhtml(string $htmlContent, string $charset = 'UTF-8'): \DOMDocument
     {
-        $htmlContent = $this->convertToHtmlEntities($htmlContent, $charset);
+        if ('UTF-8' === $charset && preg_match('//u', $htmlContent)) {
+            $htmlContent = '<?xml encoding="UTF-8">'.$htmlContent;
+        } else {
+            $htmlContent = $this->convertToHtmlEntities($htmlContent, $charset);
+        }
 
         $internalErrors = libxml_use_internal_errors(true);
 
diff --git a/Tests/AbstractCrawlerTestCase.php b/Tests/AbstractCrawlerTestCase.php
@@ -184,6 +184,10 @@ public function testAddContent()
         $crawler = $this->createCrawler();
         $crawler->addContent($this->getDoctype().'<html><meta http-equiv="Content-Type" content="text/html; charset=unicode" /><div class="foo"></html></html>');
         $this->assertEquals('foo', $crawler->filterXPath('//div')->attr('class'), '->addContent() ignores bad charset');
+
+        $crawler = $this->createCrawler();
+        $crawler->addContent($this->getDoctype().'<html><script>var foo = "bär";</script></html>', 'text/html; charset=UTF-8');
+        $this->assertEquals('var foo = "bär";', $crawler->filterXPath('//script')->text(), '->addContent() does not interfere with script content');
     }
 
     /**
