From 1be795af4772369cd1ef1d17216b652169715895 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Thu, 2 Jan 2025 14:46:27 -0500 Subject: [PATCH 1/2] [Tools.JavaSource] Support `` tags with attributes A recent attempt to import API docs for API 35 produced the following: The following issues were found, review the build log for more details: > ## Unable to translate remarks for android/app/admin/DevicePolicyManager: > JavadocImport-Error (31:39): Syntax error, expected:

,

, #PCDATA, , , , , , {@code, {@docRoot}, {@inheritDoc}, {@link, {@linkplain, {@literal, {@see, {@value}, {@value, IgnorableDeclaration, {@param, UnknownHtmlElementStart,

,

, 

A Device Owner, which only ever exists on the
                                              ^

Parsing logic fails here because the `` tag has an `id` attribute
_and_ is present in an open `
` tag:

    ParserTrace:
      input=`
 (Key symbol)`; error? False; message=Shift to S3
      input=``; error? False; message=Reduce on '
 -> 
 '
      input=`
`; error? False; message=Popped state from stack, pushing 

      input=`For  (#PCDATA)`; error? False; message=Shift to S10
      input=``; error? False; message=Reduce on ' -> #PCDATA '
      input=``; error? False; message=Popped state from stack, pushing 
      input=``; error? False; message=Reduce on 'InlineDeclaration ->  '
      input=`InlineDeclaration`; error? False; message=Popped state from stack, pushing InlineDeclaration
      input=``; error? False; message=Reduce on 'InlineDeclaration+ -> InlineDeclaration '
      input=`InlineDeclaration+`; error? False; message=Popped state from stack, pushing InlineDeclaration+
      input=`< (UnknownHtmlElementStart)`; error? False; message=Shift to S45
      input=``; error? False; message=Reduce on ' -> UnknownHtmlElementStart '
      input=``; error? False; message=Popped state from stack, pushing 
      input=``; error? False; message=Reduce on 'InlineDeclaration ->  '
      input=`InlineDeclaration`; error? False; message=Popped state from stack, pushing InlineDeclaration
      input=``; error? False; message=Reduce on 'InlineDeclaration+ -> InlineDeclaration+ InlineDeclaration '
      input=`InlineDeclaration+`; error? False; message=Popped state from stack, pushing InlineDeclaration+
      input=`li>A  (#PCDATA)`; error? False; message=Shift to S10
      input=``; error? False; message=Reduce on ' -> #PCDATA '
      input=``; error? False; message=Popped state from stack, pushing 
      input=``; error? False; message=Reduce on 'InlineDeclaration ->  '
      input=`InlineDeclaration`; error? False; message=Popped state from stack, pushing InlineDeclaration
      input=``; error? False; message=Reduce on 'InlineDeclaration+ -> InlineDeclaration+ InlineDeclaration '
      input=`InlineDeclaration+`; error? False; message=Popped state from stack, pushing InlineDeclaration+
      input=`< (UnknownHtmlElementStart)`; error? False; message=Shift to S45
      input=``; error? False; message=Reduce on ' -> UnknownHtmlElementStart '
      input=``; error? False; message=Popped state from stack, pushing 
      input=``; error? False; message=Reduce on 'InlineDeclaration ->  '
      input=`InlineDeclaration`; error? False; message=Popped state from stack, pushing InlineDeclaration
      input=``; error? False; message=Reduce on 'InlineDeclaration+ -> InlineDeclaration+ InlineDeclaration '
      input=`InlineDeclaration+`; error? False; message=Popped state from stack, pushing InlineDeclaration+
      input=`i id="deviceowner">Device Owner (#PCDATA)`; error? False; message=Shift to S10
      input=``; error? False; message=Reduce on ' -> #PCDATA '
      input=``; error? False; message=Popped state from stack, pushing 
      input=``; error? False; message=Reduce on 'InlineDeclaration ->  '
      input=`InlineDeclaration`; error? False; message=Popped state from stack, pushing InlineDeclaration
      input=``; error? False; message=Reduce on 'InlineDeclaration+ -> InlineDeclaration+ InlineDeclaration '
      input=`InlineDeclaration+`; error? False; message=Popped state from stack, pushing InlineDeclaration+
      input=` (Key symbol)`; error? False; message=Reduce on 'InlineDeclarations -> InlineDeclaration+ '
      input=`InlineDeclarations`; error? False; message=Popped state from stack, pushing InlineDeclarations
      input=` (Key symbol)`; error? True; message=Syntax error, expected: 
, 

      input=` (Key symbol)`; error? False; message=RECOVERING: popping stack, looking for state with error shift
      input=` (Key symbol)`; error? False; message=FAILED TO RECOVER

Updates the grammar to allow for `` html tags that include
attributes to fix this.

The regex in `CreateStartElementIgnoreAttribute` has also been improved
to include word boundaries around the tag name to make sure that it does
not match unexpected elements.
---
 .../SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs      | 11 +++++++----
 .../SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs |  4 ++++
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs
index 2c22a5339..5abd2bf25 100644
--- a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs
+++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs
@@ -63,7 +63,7 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 				};
 
 				var fontstyle_tt    = CreateHtmlToCrefElement (grammar, "tt",   "c", InlineDeclarations, optionalEnd: true);
-				var fontstyle_i     = CreateHtmlToCrefElement (grammar, "i",    "i", InlineDeclarations, optionalEnd: true);
+				var fontstyle_i     = CreateHtmlToCrefElement (grammar, "i",    "i", InlineDeclarations, optionalEnd: true, ignoreAttributes: true);
 
 				var preText = new PreBlockDeclarationBodyTerminal ();
 				PreBlockDeclaration.Rule = CreateStartElementIgnoreAttribute ("pre") + preText + CreateEndElement ("pre", grammar, optional: true);
@@ -258,9 +258,12 @@ static string GetChildNodesAsString (ParseTreeNode parseNode)
 				},
 			};
 
-			static NonTerminal CreateHtmlToCrefElement (Grammar grammar, string htmlElement, string crefElement, BnfTerm body, bool optionalEnd = false)
+			static NonTerminal CreateHtmlToCrefElement (Grammar grammar, string htmlElement, string crefElement, BnfTerm body, bool optionalEnd = false, bool ignoreAttributes = false)
 			{
-				var start       = CreateStartElement (htmlElement, grammar);
+				BnfTerm start   = CreateStartElement (htmlElement, grammar);
+				if (ignoreAttributes) {
+					start = CreateStartElementIgnoreAttribute (htmlElement);
+				}
 				var end         = CreateEndElement (htmlElement, grammar, optionalEnd);
 				var nonTerminal = new NonTerminal ("<" + htmlElement + ">", ConcatChildNodes) {
 					Rule = start + body + end,
@@ -285,7 +288,7 @@ static NonTerminal CreateStartElement (string startElement, Grammar grammar)
 
 			static RegexBasedTerminal CreateStartElementIgnoreAttribute (string startElement, string attribute)
 			{
-				return new RegexBasedTerminal ($"<{startElement} {attribute}", $@"(?i)<{startElement}\s*{attribute}[^>]*>") {
+				return new RegexBasedTerminal ($"<{startElement} {attribute}", $@"(?i)<\b{startElement}\b\s*{attribute}[^>]*>") {
 					AstConfig = new AstNodeConfig {
 						NodeCreator = (context, parseNode) => parseNode.AstNode = "",
 					},
diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs
index fb74b4845..5f5061f2d 100644
--- a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs
+++ b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs
@@ -34,6 +34,10 @@ public void PBlockDeclaration ()
 			r = p.Parse("
r= unknown text");
 			Assert.IsFalse (r.HasErrors (), DumpMessages (r, p));
 			Assert.AreEqual ("r= <em>unknown</em> text", r.Root.AstNode.ToString ());
+
+			r = p.Parse ("
For 
  • A Device Owner");
+			Assert.IsFalse (r.HasErrors (), DumpMessages (r, p));
+			Assert.AreEqual ("For <li>A Device Owner", r.Root.AstNode.ToString ());
 		}
 
 		[Test]

From 0ca8d8d5bf91e03d4301c67c1a81ea34ad999adf Mon Sep 17 00:00:00 2001
From: Peter Collins 
Date: Mon, 6 Jan 2025 13:37:41 -0500
Subject: [PATCH 2/2] Apply feedback

---
 ...urceJavadocToXmldocGrammar.HtmlBnfTerms.cs | 30 +++++--------------
 1 file changed, 7 insertions(+), 23 deletions(-)

diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs
index 5abd2bf25..82abd11c7 100644
--- a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs
+++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs
@@ -63,10 +63,10 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 				};
 
 				var fontstyle_tt    = CreateHtmlToCrefElement (grammar, "tt",   "c", InlineDeclarations, optionalEnd: true);
-				var fontstyle_i     = CreateHtmlToCrefElement (grammar, "i",    "i", InlineDeclarations, optionalEnd: true, ignoreAttributes: true);
+				var fontstyle_i     = CreateHtmlToCrefElement (grammar, "i",    "i", InlineDeclarations, optionalEnd: true);
 
 				var preText = new PreBlockDeclarationBodyTerminal ();
-				PreBlockDeclaration.Rule = CreateStartElementIgnoreAttribute ("pre") + preText + CreateEndElement ("pre", grammar, optional: true);
+				PreBlockDeclaration.Rule = CreateStartElement ("pre") + preText + CreateEndElement ("pre", grammar, optional: true);
 				PreBlockDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
 					if (!grammar.ShouldImport (ImportJavadoc.Remarks)) {
 						parseNode.AstNode   = "";
@@ -82,7 +82,7 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar)
 				FontStyleDeclaration.Rule = fontstyle_tt | fontstyle_i;
 
 				PBlockDeclaration.Rule =
-					CreateStartElement ("p", grammar) + InlineDeclarations + CreateEndElement ("p", grammar, optional:true)
+					CreateStartElement ("p") + InlineDeclarations + CreateEndElement ("p", grammar, optional:true)
 					;
 				PBlockDeclaration.AstConfig.NodeCreator = (context, parseNode) => {
 					var remarks     = FinishParse (context, parseNode).Remarks;
@@ -258,12 +258,9 @@ static string GetChildNodesAsString (ParseTreeNode parseNode)
 				},
 			};
 
-			static NonTerminal CreateHtmlToCrefElement (Grammar grammar, string htmlElement, string crefElement, BnfTerm body, bool optionalEnd = false, bool ignoreAttributes = false)
+			static NonTerminal CreateHtmlToCrefElement (Grammar grammar, string htmlElement, string crefElement, BnfTerm body, bool optionalEnd = false)
 			{
-				BnfTerm start   = CreateStartElement (htmlElement, grammar);
-				if (ignoreAttributes) {
-					start = CreateStartElementIgnoreAttribute (htmlElement);
-				}
+				var start       = CreateStartElement (htmlElement);
 				var end         = CreateEndElement (htmlElement, grammar, optionalEnd);
 				var nonTerminal = new NonTerminal ("<" + htmlElement + ">", ConcatChildNodes) {
 					Rule = start + body + end,
@@ -278,28 +275,15 @@ static NonTerminal CreateHtmlToCrefElement (Grammar grammar, string htmlElement,
 				return nonTerminal;
 			}
 
-			static NonTerminal CreateStartElement (string startElement, Grammar grammar)
-			{
-				var start   = new NonTerminal ("<" + startElement + ">", nodeCreator: (context, parseNode) => parseNode.AstNode = "") {
-					Rule    = grammar.ToTerm ("<" + startElement + ">") | "<" + startElement.ToUpperInvariant () + ">",
-				};
-				return start;
-			}
-
-			static RegexBasedTerminal CreateStartElementIgnoreAttribute (string startElement, string attribute)
+			static RegexBasedTerminal CreateStartElement (string startElement, string attribute = "")
 			{
-				return new RegexBasedTerminal ($"<{startElement} {attribute}", $@"(?i)<\b{startElement}\b\s*{attribute}[^>]*>") {
+				return new RegexBasedTerminal ($"<{startElement} {attribute}>", $@"(?i)<\b{startElement}\b\s*{attribute}[^>]*>") {
 					AstConfig = new AstNodeConfig {
 						NodeCreator = (context, parseNode) => parseNode.AstNode = "",
 					},
 				};
 			}
 
-			static RegexBasedTerminal CreateStartElementIgnoreAttribute (string startElement)
-			{
-				return CreateStartElementIgnoreAttribute (startElement, "");
-			}
-
 			static NonTerminal CreateEndElement (string endElement, Grammar grammar, bool optional = false)
 			{
 				var end	    = new NonTerminal (endElement, nodeCreator: (context, parseNode) => parseNode.AstNode = "") {