From d442c155cd60ca4977a2e5f7a658e675bbd61862 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Tue, 13 Jun 2023 18:06:57 -0400 Subject: [PATCH 1/5] [Java.Interop.Tools.JavaSource] Fix up common HTML tags Fixes: https://github.com/xamarin/java.interop/issues/1071 The latest API docs update contained a couple dozen parsing issues due to broken `` elements, reserved inline characters in `` elements, and other issues. These issues have been fixed by no longer attempting to parse `` elements with Irony. Instead, an HTML processing step has been added which replaces, removes, or decodes well known HTML tags after the javadoc is parsed. Parsing for `` elements has also been updated to fix all 83 cases where `href` attribute parsing would fail. Now when we we encounter an `` element that points to code or a local path we will only include the element value in the javadoc, and not the full `href` attribute. Readability of our generated docs should be improved by both of these changes, as there will be fewer encoded character entities in places where they are not necessary. --- ...urceJavadocToXmldocGrammar.HtmlBnfTerms.cs | 87 ++++++++++--------- ...vadocToXmldocGrammar.InlineTagsBnfTerms.cs | 43 ++++++--- ...avadocToXmldocGrammar.HtmlBnfTermsTests.cs | 15 +--- .../SourceJavadocToXmldocParserTests.cs | 8 +- .../JavadocInfo.cs | 39 +++++++++ 5 files changed, 122 insertions(+), 70 deletions(-) diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs index f17b63600..347f50e63 100644 --- a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs @@ -16,6 +16,7 @@ namespace Java.Interop.Tools.JavaSource { public partial class SourceJavadocToXmldocGrammar { public class HtmlBnfTerms { + internal HtmlBnfTerms () { } @@ -25,7 +26,6 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) AllHtmlTerms.Rule = TopLevelInlineDeclaration | PBlockDeclaration | PreBlockDeclaration - | IgnorableElementDeclaration ; var inlineDeclaration = new NonTerminal ("", ConcatChildNodes) { @@ -37,7 +37,6 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) | FormCtrlDeclaration */ | InlineHyperLinkDeclaration - | CodeElementDeclaration | grammar.InlineTagsTerms.AllInlineTerms | UnknownHtmlElementStart , @@ -100,48 +99,53 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) parseNode.AstNode = p; }; - InlineHyperLinkDeclaration.Rule = InlineHyperLinkOpenTerm + InlineDeclarations + CreateEndElement ("a", grammar, optional: true); + InlineHyperLinkDeclaration.Rule = HtmlAElementStart + InlineDeclarations + CreateEndElement ("a", grammar, optional: true); InlineHyperLinkDeclaration.AstConfig.NodeCreator = (context, parseNode) => { - var unparsedAElementValue = string.Empty; - foreach (var cn in parseNode.ChildNodes) { - if (cn.ChildNodes?.Count > 1) { - foreach (var gcn in cn.ChildNodes) { - unparsedAElementValue += gcn.AstNode?.ToString (); - } - } else { - unparsedAElementValue += cn.AstNode?.ToString (); - } - } + var nodesAsString = GetChildNodesAsString (parseNode); + var tokenValue = parseNode.ChildNodes [0].Token.Text; + int stopIndex = nodesAsString.IndexOf ('>'); - var seeElement = TryParseHRef (unparsedAElementValue); - if (seeElement == null) - seeElement = TryParseHRef (WebUtility.HtmlDecode (unparsedAElementValue), logError: true); + if (stopIndex == -1 || !tokenValue.Contains ("href", StringComparison.OrdinalIgnoreCase)) { + parseNode.AstNode = new XText (nodesAsString); + return; + } - var hrefValue = seeElement?.Attribute ("href")?.Value ?? string.Empty; - if (!string.IsNullOrEmpty (hrefValue) && - (hrefValue.StartsWith ("http", StringComparison.OrdinalIgnoreCase) || hrefValue.StartsWith ("www", StringComparison.OrdinalIgnoreCase))) { - parseNode.AstNode = seeElement; + var attributeName = parseNode.ChildNodes [0].Term.Name; + var attributeValue = nodesAsString.Substring (0, stopIndex).Trim ().Trim('\'', '"'); + var elementValue = nodesAsString.Substring (stopIndex + 1); + if (!string.IsNullOrEmpty (attributeValue) && + (attributeValue.StartsWith ("http", StringComparison.OrdinalIgnoreCase) || attributeValue.StartsWith ("www", StringComparison.OrdinalIgnoreCase))) { + var unparsed = $"{elementValue}"; + XNode? seeElement = TryParseElement (unparsed); + if (seeElement == null) { + // Try to parse with HTML entities decoded + seeElement = TryParseElement (WebUtility.HtmlDecode (unparsed)); + if (seeElement == null) { + // Finally, try to parse with only the element value encoded + seeElement = TryParseElement ($"{WebUtility.HtmlEncode (elementValue)}", logError: true); + } + } + parseNode.AstNode = seeElement ?? new XText (nodesAsString); } else { // TODO: Need to convert relative paths or code references to appropriate CREF value. - parseNode.AstNode = new XText (unparsedAElementValue); + parseNode.AstNode = new XText (elementValue); } }; + } - // Start to trim out unusable HTML elements/tags, but not any inner values - IgnorableElementDeclaration.Rule = - CreateStartElementIgnoreAttribute ("a", "name") + InlineDeclarations + CreateEndElement ("a", grammar, optional: true) - | CreateStartElementIgnoreAttribute ("a", "id") + InlineDeclarations + CreateEndElement ("a", grammar, optional: true) - ; - IgnorableElementDeclaration.AstConfig.NodeCreator = (context, parseNode) => { - var aElementValue = new XText (parseNode.ChildNodes [1].AstNode.ToString () ?? string.Empty); - parseNode.AstNode = aElementValue; - }; - - CodeElementDeclaration.Rule = CreateStartElement ("code", grammar) + InlineDeclarations + CreateEndElement ("code", grammar); - CodeElementDeclaration.AstConfig.NodeCreator = (context, parseNode) => { - var target = parseNode.ChildNodes [1].AstNode; - parseNode.AstNode = new XElement ("c", target); - }; + static string GetChildNodesAsString (ParseTreeNode parseNode) + { + var unparsed = string.Empty; + foreach (var cn in parseNode.ChildNodes) { + if (cn.ChildNodes?.Count > 1) { + foreach (var gcn in cn.ChildNodes) { + unparsed += gcn.AstNode?.ToString (); + } + } else { + unparsed += cn.AstNode?.ToString (); + } + } + return unparsed; } static IEnumerable GetParagraphs (ParseTreeNodeList children) @@ -184,13 +188,13 @@ static IEnumerable GetParagraphs (ParseTreeNodeList children) } } - static XElement? TryParseHRef (string unparsedAElementValue, bool logError = false) + static XElement? TryParseElement (string unparsed, bool logError = false) { try { - return XElement.Parse ($""); + return XElement.Parse (unparsed); } catch (Exception x) { if (logError) - Console.Error.WriteLine ($"## Unable to parse HTML element: \n{x.GetType ()}: {x.Message}"); + Console.Error.WriteLine ($"## Unable to parse HTML element: `{unparsed}`\n{x.GetType ()}: {x.Message}"); return null; } } @@ -221,15 +225,12 @@ static IEnumerable GetParagraphs (ParseTreeNodeList children) public readonly NonTerminal PBlockDeclaration = new NonTerminal (nameof (PBlockDeclaration), ConcatChildNodes); public readonly NonTerminal PreBlockDeclaration = new NonTerminal (nameof (PreBlockDeclaration), ConcatChildNodes); public readonly NonTerminal InlineHyperLinkDeclaration = new NonTerminal (nameof (InlineHyperLinkDeclaration), ConcatChildNodes); - public readonly NonTerminal IgnorableElementDeclaration = new NonTerminal (nameof (IgnorableElementDeclaration), ConcatChildNodes); - public readonly NonTerminal CodeElementDeclaration = new NonTerminal (nameof (CodeElementDeclaration), ConcatChildNodes); - public readonly Terminal InlineHyperLinkOpenTerm = new RegexBasedTerminal (" parseNode.AstNode = parseNode.Token.Value.ToString (), diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs index f78fab170..cf7efe28e 100644 --- a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs @@ -29,8 +29,8 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) | LiteralDeclaration | SeeDeclaration | ValueDeclaration - | IgnorableDeclaration | InlineParamDeclaration + | IgnorableDeclaration ; CodeDeclaration.Rule = grammar.ToTerm ("{@code") + InlineValue + "}"; @@ -109,15 +109,6 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) } }; - // Inline content may contain reserved characters with no tags or special parsing rules, do not throw when encountering them - IgnorableDeclaration.Rule = grammar.ToTerm ("@ ") - | grammar.ToTerm ("{") - | grammar.ToTerm ("}") - ; - IgnorableDeclaration.AstConfig.NodeCreator = (context, parseNode) => { - parseNode.AstNode = new XText (parseNode.ChildNodes [0].Term.Name.Trim ()); - }; - InlineParamDeclaration.Rule = grammar.ToTerm ("{@param") + InlineValue + "}"; InlineParamDeclaration.AstConfig.NodeCreator = (context, parseNode) => { var target = parseNode.ChildNodes [1].AstNode; @@ -156,9 +147,37 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) // https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#value public readonly NonTerminal ValueDeclaration = new NonTerminal (nameof (ValueDeclaration)); - public readonly NonTerminal IgnorableDeclaration = new NonTerminal (nameof (IgnorableDeclaration)); - public readonly NonTerminal InlineParamDeclaration = new NonTerminal (nameof (InlineParamDeclaration)); + + public readonly Terminal IgnorableDeclaration = new IgnorableCharTerminal (nameof (IgnorableDeclaration)) { + AstConfig = new AstNodeConfig { + NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value.ToString (), + }, + }; + + } + } + + class IgnorableCharTerminal : Terminal + { + public IgnorableCharTerminal (string name) + : base (name) + { + Priority = TerminalPriority.Low - 1; } + + public override Token? TryMatch (ParsingContext context, ISourceStream source) + { + var startChar = source.Text [source.Location.Position]; + if (startChar != '@' + && startChar != '{' + && startChar != '}' + ) { + return null; + } + source.PreviewPosition += 1; + return source.CreateToken (OutputTerminal, startChar); + } + } } diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs index eaef58232..f63c43e51 100644 --- a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs +++ b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs @@ -69,19 +69,12 @@ public void HyperLinkDeclaration () r = p.Parse ("field classification"); Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); - Assert.AreEqual ("\"AutofillService.html#FieldClassification\">field classification", - r.Root.AstNode.ToString ()); - } + Assert.AreEqual ("field classification", r.Root.AstNode.ToString ()); - [Test] - public void CodeElementDeclaration () - { - var p = CreateParser (g => g.HtmlTerms.CodeElementDeclaration); - - var r = p.Parse ("input.position()"); + r = p.Parse ("\nProgress & activity"); Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); - Assert.AreEqual ("input.position()", r.Root.AstNode.ToString ()); + Assert.AreEqual ("\nProgress & activity", + r.Root.AstNode.ToString ()); } - } } diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs index d224a6ee9..d57328967 100644 --- a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs +++ b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs @@ -144,16 +144,16 @@ more description here. ", }, new ParseResult { - Javadoc = "Something {@link #method}: description, \"declaration\" or \"another declaration\".\n\n@apiSince 1\n", + Javadoc = "Something {@link #method}: description.\n\n@apiSince 1\n", FullXml = @" - Something #method: description, ""declaration"" or ""another declaration"". + Something #method: description. - Something #method: description, ""declaration"" or ""another declaration"". + Something #method: description. Added in API level 1. ", IntelliSenseXml = @" - Something #method: description, ""declaration"" or ""another declaration"". + Something #method: description. ", }, new ParseResult { diff --git a/tools/generator/Java.Interop.Tools.Generator.ObjectModel/JavadocInfo.cs b/tools/generator/Java.Interop.Tools.Generator.ObjectModel/JavadocInfo.cs index c142a930c..bd66c284b 100644 --- a/tools/generator/Java.Interop.Tools.Generator.ObjectModel/JavadocInfo.cs +++ b/tools/generator/Java.Interop.Tools.Generator.ObjectModel/JavadocInfo.cs @@ -25,6 +25,7 @@ public sealed class JavadocInfo { public XElement[] Copyright { get; set; } public XmldocStyle XmldocStyle { get; set; } + public string DocRootReplacement { get; set; } string MemberDescription; @@ -202,6 +203,7 @@ static void AddNode (ICollection comments, XNode node) if (node == null) return; var contents = node.ToString (); + contents = FixUpHtml (contents); var lines = new StringReader (contents); string line; @@ -210,6 +212,43 @@ static void AddNode (ICollection comments, XNode node) } } + // Remove, replace, or decode common HTML tags to improve what is displayed in the IDE and online. + static string FixUpHtml (string javadocContent) + { + var replacements = new Dictionary { + { "<blockquote>", "" }, { "</blockquote>", "" }, + { "<cite>", "" }, { "</cite>", "" }, + { "<code>", "" }, { "</code>", "" }, + { "<dd>", "" }, { "</dd>", "" }, + { "<dl>", "" }, { "</dl>", "" }, + { "<dt>", "" }, { "</dt>", "" }, + { "<em>", "" }, { "</em>", "" }, + { "<h1>", "" }, { "</h1>", "" }, + { "<h2>", "" }, { "</h2>", "" }, + { "<h3>", "" }, { "</h3>", "" }, + { "<h4>", "" }, { "</h4>", "" }, + { "<h5>", "" }, { "</h5>", "" }, + { "<h6>", "" }, { "</h6>", "" }, + { "<li>", "" }, { "</li>", "" }, + { "<ol>", "" }, { "</ol>", "" }, + { "<strong>", "" }, { "</strong>", "" }, + { "<sub>", "" }, { "</sub>", "" }, + { "<sup>", "" }, { "</sup>", "" }, + { "<table", "" }, + { "<tbody>", "" }, { "</tbody>", "" }, + { "<td>", "" }, { "</td>", "" }, + { "<th>", "" }, { "</th>", "" }, + { "<thead>", "" }, { "</thead>", "" }, + { "<tr>", "" }, { "</tr>", "" }, + { "<ul>", "" }, { "</ul>", "" }, + }; + + foreach (var r in replacements) { + javadocContent = javadocContent.Replace (r.Key, r.Value); + } + return javadocContent; + } + static void PrintMessages (ParseTree tree, TextWriter writer) { var lines = GetLines (tree.SourceText); From 945535d9d2327371a05216ab6031f81fcbfea1a1 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Tue, 13 Jun 2023 20:08:39 -0400 Subject: [PATCH 2/5] Fix test --- .../SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs index f63c43e51..8df26781e 100644 --- a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs +++ b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs @@ -73,7 +73,7 @@ public void HyperLinkDeclaration () r = p.Parse ("\nProgress & activity"); Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); - Assert.AreEqual ("\nProgress & activity", + Assert.AreEqual ($"{Environment.NewLine}Progress & activity", r.Root.AstNode.ToString ()); } } From a23212720943358a6bd8a7b96340c2b6e44b1e99 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Tue, 11 Jul 2023 16:44:12 -0400 Subject: [PATCH 3/5] Apply feedback --- ...urceJavadocToXmldocGrammar.HtmlBnfTerms.cs | 99 +++++++++++-------- ...vadocToXmldocGrammar.InlineTagsBnfTerms.cs | 43 +++----- ...avadocToXmldocGrammar.HtmlBnfTermsTests.cs | 46 ++++++++- .../SourceJavadocToXmldocParserTests.cs | 8 +- .../JavadocInfo.cs | 39 -------- 5 files changed, 114 insertions(+), 121 deletions(-) diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs index 347f50e63..252403387 100644 --- a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs @@ -16,7 +16,6 @@ namespace Java.Interop.Tools.JavaSource { public partial class SourceJavadocToXmldocGrammar { public class HtmlBnfTerms { - internal HtmlBnfTerms () { } @@ -26,6 +25,7 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) AllHtmlTerms.Rule = TopLevelInlineDeclaration | PBlockDeclaration | PreBlockDeclaration + | IgnorableElementDeclaration ; var inlineDeclaration = new NonTerminal ("", ConcatChildNodes) { @@ -37,6 +37,7 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) | FormCtrlDeclaration */ | InlineHyperLinkDeclaration + | CodeElementDeclaration | grammar.InlineTagsTerms.AllInlineTerms | UnknownHtmlElementStart , @@ -99,53 +100,56 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) parseNode.AstNode = p; }; - InlineHyperLinkDeclaration.Rule = HtmlAElementStart + InlineDeclarations + CreateEndElement ("a", grammar, optional: true); + InlineHyperLinkDeclaration.Rule = InlineHyperLinkOpenTerm + InlineDeclarations + CreateEndElement ("a", grammar, optional: true); InlineHyperLinkDeclaration.AstConfig.NodeCreator = (context, parseNode) => { - var nodesAsString = GetChildNodesAsString (parseNode); - var tokenValue = parseNode.ChildNodes [0].Token.Text; - int stopIndex = nodesAsString.IndexOf ('>'); - - if (stopIndex == -1 || !tokenValue.Contains ("href", StringComparison.OrdinalIgnoreCase)) { - parseNode.AstNode = new XText (nodesAsString); - return; - } - - var attributeName = parseNode.ChildNodes [0].Term.Name; - var attributeValue = nodesAsString.Substring (0, stopIndex).Trim ().Trim('\'', '"'); - var elementValue = nodesAsString.Substring (stopIndex + 1); - if (!string.IsNullOrEmpty (attributeValue) && - (attributeValue.StartsWith ("http", StringComparison.OrdinalIgnoreCase) || attributeValue.StartsWith ("www", StringComparison.OrdinalIgnoreCase))) { - var unparsed = $"{elementValue}"; - XNode? seeElement = TryParseElement (unparsed); - if (seeElement == null) { - // Try to parse with HTML entities decoded - seeElement = TryParseElement (WebUtility.HtmlDecode (unparsed)); - if (seeElement == null) { - // Finally, try to parse with only the element value encoded - seeElement = TryParseElement ($"{WebUtility.HtmlEncode (elementValue)}", logError: true); + var unparsedAElementValue = string.Empty; + foreach (var cn in parseNode.ChildNodes) { + if (cn.ChildNodes?.Count > 1) { + foreach (var gcn in cn.ChildNodes) { + unparsedAElementValue += gcn.AstNode?.ToString (); } + } else { + unparsedAElementValue += cn.AstNode?.ToString (); } - parseNode.AstNode = seeElement ?? new XText (nodesAsString); + } + + var seeElement = TryParseHRef (unparsedAElementValue); + if (seeElement == null) + seeElement = TryParseHRef (WebUtility.HtmlDecode (unparsedAElementValue), logError: true); + + var hrefValue = seeElement?.Attribute ("href")?.Value ?? string.Empty; + if (!string.IsNullOrEmpty (hrefValue) && + (hrefValue.StartsWith ("http", StringComparison.OrdinalIgnoreCase) || hrefValue.StartsWith ("www", StringComparison.OrdinalIgnoreCase))) { + parseNode.AstNode = seeElement; } else { // TODO: Need to convert relative paths or code references to appropriate CREF value. - parseNode.AstNode = new XText (elementValue); + parseNode.AstNode = new XText (unparsedAElementValue); } }; - } - static string GetChildNodesAsString (ParseTreeNode parseNode) - { - var unparsed = string.Empty; - foreach (var cn in parseNode.ChildNodes) { - if (cn.ChildNodes?.Count > 1) { - foreach (var gcn in cn.ChildNodes) { - unparsed += gcn.AstNode?.ToString (); - } - } else { - unparsed += cn.AstNode?.ToString (); + // Start to trim out unusable HTML elements/tags, but not any inner values + IgnorableElementDeclaration.Rule = + CreateStartElementIgnoreAttribute ("a", "name") + InlineDeclarations + CreateEndElement ("a", grammar, optional: true) + | CreateStartElementIgnoreAttribute ("a", "id") + InlineDeclarations + CreateEndElement ("a", grammar, optional: true) + ; + IgnorableElementDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + var aElementValue = new XText (parseNode.ChildNodes [1].AstNode.ToString () ?? string.Empty); + parseNode.AstNode = aElementValue; + }; + + CodeElementDeclaration.Rule = CodeElementContentTerm; + CodeElementDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + // Parse the entire element captured in the token + var codeElementText = parseNode.ChildNodes [0].Token.Text; + int startIndex = codeElementText.IndexOf ('>'); + int stopIndex = codeElementText.LastIndexOf ('<'); + if (startIndex == -1 || stopIndex == -1) { + parseNode.AstNode = new XText (codeElementText); + return; } - } - return unparsed; + var target = codeElementText.Substring (startIndex + 1, stopIndex - startIndex - 1); + parseNode.AstNode = new XElement ("c", target); + }; } static IEnumerable GetParagraphs (ParseTreeNodeList children) @@ -188,13 +192,13 @@ static IEnumerable GetParagraphs (ParseTreeNodeList children) } } - static XElement? TryParseElement (string unparsed, bool logError = false) + static XElement? TryParseHRef (string unparsedAElementValue, bool logError = false) { try { - return XElement.Parse (unparsed); + return XElement.Parse ($""); } catch (Exception x) { if (logError) - Console.Error.WriteLine ($"## Unable to parse HTML element: `{unparsed}`\n{x.GetType ()}: {x.Message}"); + Console.Error.WriteLine ($"## Unable to parse HTML element: \n{x.GetType ()}: {x.Message}"); return null; } } @@ -225,12 +229,21 @@ static IEnumerable GetParagraphs (ParseTreeNodeList children) public readonly NonTerminal PBlockDeclaration = new NonTerminal (nameof (PBlockDeclaration), ConcatChildNodes); public readonly NonTerminal PreBlockDeclaration = new NonTerminal (nameof (PreBlockDeclaration), ConcatChildNodes); public readonly NonTerminal InlineHyperLinkDeclaration = new NonTerminal (nameof (InlineHyperLinkDeclaration), ConcatChildNodes); + public readonly NonTerminal IgnorableElementDeclaration = new NonTerminal (nameof (IgnorableElementDeclaration), ConcatChildNodes); + public readonly NonTerminal CodeElementDeclaration = new NonTerminal (nameof (CodeElementDeclaration), ConcatChildNodes); - public readonly Terminal HtmlAElementStart = new RegexBasedTerminal ("", $@"(?i)]*>(.|\s)*?(<\/code>|<\/null>|)") { AstConfig = new AstNodeConfig { NodeCreator = (context, parseNode) => parseNode.AstNode = "", }, }; + + public readonly Terminal InlineHyperLinkOpenTerm = new RegexBasedTerminal (" parseNode.AstNode = parseNode.Token.Value.ToString (), diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs index cf7efe28e..f78fab170 100644 --- a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.InlineTagsBnfTerms.cs @@ -29,8 +29,8 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) | LiteralDeclaration | SeeDeclaration | ValueDeclaration - | InlineParamDeclaration | IgnorableDeclaration + | InlineParamDeclaration ; CodeDeclaration.Rule = grammar.ToTerm ("{@code") + InlineValue + "}"; @@ -109,6 +109,15 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) } }; + // Inline content may contain reserved characters with no tags or special parsing rules, do not throw when encountering them + IgnorableDeclaration.Rule = grammar.ToTerm ("@ ") + | grammar.ToTerm ("{") + | grammar.ToTerm ("}") + ; + IgnorableDeclaration.AstConfig.NodeCreator = (context, parseNode) => { + parseNode.AstNode = new XText (parseNode.ChildNodes [0].Term.Name.Trim ()); + }; + InlineParamDeclaration.Rule = grammar.ToTerm ("{@param") + InlineValue + "}"; InlineParamDeclaration.AstConfig.NodeCreator = (context, parseNode) => { var target = parseNode.ChildNodes [1].AstNode; @@ -147,37 +156,9 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) // https://docs.oracle.com/javase/7/docs/technotes/tools/windows/javadoc.html#value public readonly NonTerminal ValueDeclaration = new NonTerminal (nameof (ValueDeclaration)); - public readonly NonTerminal InlineParamDeclaration = new NonTerminal (nameof (InlineParamDeclaration)); - - public readonly Terminal IgnorableDeclaration = new IgnorableCharTerminal (nameof (IgnorableDeclaration)) { - AstConfig = new AstNodeConfig { - NodeCreator = (context, parseNode) => parseNode.AstNode = parseNode.Token.Value.ToString (), - }, - }; - - } - } + public readonly NonTerminal IgnorableDeclaration = new NonTerminal (nameof (IgnorableDeclaration)); - class IgnorableCharTerminal : Terminal - { - public IgnorableCharTerminal (string name) - : base (name) - { - Priority = TerminalPriority.Low - 1; - } - - public override Token? TryMatch (ParsingContext context, ISourceStream source) - { - var startChar = source.Text [source.Location.Position]; - if (startChar != '@' - && startChar != '{' - && startChar != '}' - ) { - return null; - } - source.PreviewPosition += 1; - return source.CreateToken (OutputTerminal, startChar); + public readonly NonTerminal InlineParamDeclaration = new NonTerminal (nameof (InlineParamDeclaration)); } - } } diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs index 8df26781e..37671b4ca 100644 --- a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs +++ b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocGrammar.HtmlBnfTermsTests.cs @@ -69,12 +69,50 @@ public void HyperLinkDeclaration () r = p.Parse ("field classification"); Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); - Assert.AreEqual ("field classification", r.Root.AstNode.ToString ()); + Assert.AreEqual ("\"AutofillService.html#FieldClassification\">field classification", + r.Root.AstNode.ToString ()); + } + + [Test] + public void CodeElementDeclaration () + { + var p = CreateParser (g => g.HtmlTerms.CodeElementDeclaration); - r = p.Parse ("\nProgress & activity"); + var r = p.Parse ("input.position()"); Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); - Assert.AreEqual ($"{Environment.NewLine}Progress & activity", - r.Root.AstNode.ToString ()); + Assert.AreEqual ("input.position()", r.Root.AstNode.ToString ()); + + r = p.Parse ("null"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("null", r.Root.AstNode.ToString ()); + + r = p.Parse ("android:label=\"@string/resolve_title\""); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("android:label=\"@string/resolve_title\"", r.Root.AstNode.ToString ()); + + r = p.Parse ("Activity.RESULT_OK"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("Activity.RESULT_OK", r.Root.AstNode.ToString ()); + + r = p.Parse ("format.setString(MediaFormat.KEY_FRAME_RATE, null)"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual ("format.setString(MediaFormat.KEY_FRAME_RATE, null)", r.Root.AstNode.ToString ()); + + r = p.Parse (@" +

[ 0, 0, 0, 0, 0 ] +

[ 0, 0, 0, 0, 0 ] +

[ 0, 0, 1, 0, 0 ] +

[ 0, 0, 0, 0, 0 ] +

[ 0, 0, 0, 0, 0 ] +"); + Assert.IsFalse (r.HasErrors (), DumpMessages (r, p)); + Assert.AreEqual (@" +<p> [ 0, 0, 0, 0, 0 ] +<p> [ 0, 0, 0, 0, 0 ] +<p> [ 0, 0, 1, 0, 0 ] +<p> [ 0, 0, 0, 0, 0 ] +<p> [ 0, 0, 0, 0, 0 ] +", r.Root.AstNode.ToString ()); } } } diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs index d57328967..e764e9f31 100644 --- a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs +++ b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs @@ -144,16 +144,16 @@ more description here. ", }, new ParseResult { - Javadoc = "Something {@link #method}: description.\n\n@apiSince 1\n", + Javadoc = "Something {@link #method}: description, \"declaration\" or 

some content

.\n\n@apiSince 1\n", FullXml = @" - Something #method: description. + Something #method: description, ""declaration"" or <pre><p>some content</pre></p>. - Something #method: description. + Something #method: description, ""declaration"" or <pre><p>some content</pre></p>. Added in API level 1. ", IntelliSenseXml = @" - Something #method: description. + Something #method: description, ""declaration"" or <pre><p>some content</pre></p>. ", }, new ParseResult { diff --git a/tools/generator/Java.Interop.Tools.Generator.ObjectModel/JavadocInfo.cs b/tools/generator/Java.Interop.Tools.Generator.ObjectModel/JavadocInfo.cs index bd66c284b..c142a930c 100644 --- a/tools/generator/Java.Interop.Tools.Generator.ObjectModel/JavadocInfo.cs +++ b/tools/generator/Java.Interop.Tools.Generator.ObjectModel/JavadocInfo.cs @@ -25,7 +25,6 @@ public sealed class JavadocInfo { public XElement[] Copyright { get; set; } public XmldocStyle XmldocStyle { get; set; } - public string DocRootReplacement { get; set; } string MemberDescription; @@ -203,7 +202,6 @@ static void AddNode (ICollection comments, XNode node) if (node == null) return; var contents = node.ToString (); - contents = FixUpHtml (contents); var lines = new StringReader (contents); string line; @@ -212,43 +210,6 @@ static void AddNode (ICollection comments, XNode node) } } - // Remove, replace, or decode common HTML tags to improve what is displayed in the IDE and online. - static string FixUpHtml (string javadocContent) - { - var replacements = new Dictionary { - { "<blockquote>", "" }, { "</blockquote>", "" }, - { "<cite>", "" }, { "</cite>", "" }, - { "<code>", "" }, { "</code>", "" }, - { "<dd>", "" }, { "</dd>", "" }, - { "<dl>", "" }, { "</dl>", "" }, - { "<dt>", "" }, { "</dt>", "" }, - { "<em>", "" }, { "</em>", "" }, - { "<h1>", "" }, { "</h1>", "" }, - { "<h2>", "" }, { "</h2>", "" }, - { "<h3>", "" }, { "</h3>", "" }, - { "<h4>", "" }, { "</h4>", "" }, - { "<h5>", "" }, { "</h5>", "" }, - { "<h6>", "" }, { "</h6>", "" }, - { "<li>", "" }, { "</li>", "" }, - { "<ol>", "" }, { "</ol>", "" }, - { "<strong>", "" }, { "</strong>", "" }, - { "<sub>", "" }, { "</sub>", "" }, - { "<sup>", "" }, { "</sup>", "" }, - { "<table", "" }, - { "<tbody>", "" }, { "</tbody>", "" }, - { "<td>", "" }, { "</td>", "" }, - { "<th>", "" }, { "</th>", "" }, - { "<thead>", "" }, { "</thead>", "" }, - { "<tr>", "" }, { "</tr>", "" }, - { "<ul>", "" }, { "</ul>", "" }, - }; - - foreach (var r in replacements) { - javadocContent = javadocContent.Replace (r.Key, r.Value); - } - return javadocContent; - } - static void PrintMessages (ParseTree tree, TextWriter writer) { var lines = GetLines (tree.SourceText); From c8ef42dd1dbf73a232fd972a2ca8cd39a3ff25a3 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Wed, 12 Jul 2023 14:04:12 -0400 Subject: [PATCH 4/5] Add test for trailing open code tag --- .../SourceJavadocToXmldocParserTests.cs | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs index e764e9f31..a586f744d 100644 --- a/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs +++ b/tests/Java.Interop.Tools.JavaSource-Tests/SourceJavadocToXmldocParserTests.cs @@ -154,6 +154,26 @@ more description here. ", IntelliSenseXml = @" Something #method: description, ""declaration"" or <pre><p>some content</pre></p>. +", + }, + new ParseResult { + Javadoc = @"The result code will be Activity.RESULT_OK for success, + or one of these errors: + RESULT_ERROR_GENERIC_FAILURE", + FullXml = @" + The result code will be Activity.RESULT_OK for success, + or one of these errors: + RESULT_ERROR_GENERIC_FAILURE + + The result code will be Activity.RESULT_OK for success, + or one of these errors: + RESULT_ERROR_GENERIC_FAILURE + +", + IntelliSenseXml = @" + The result code will be Activity.RESULT_OK for success, + or one of these errors: + RESULT_ERROR_GENERIC_FAILURE ", }, new ParseResult { From 00896fbd5300791665a5a9e6b3755efdb191df41 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Wed, 12 Jul 2023 14:07:41 -0400 Subject: [PATCH 5/5] Fix merge --- .../SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs index d49604a30..2c22a5339 100644 --- a/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs +++ b/src/Java.Interop.Tools.JavaSource/Java.Interop.Tools.JavaSource/SourceJavadocToXmldocGrammar.HtmlBnfTerms.cs @@ -132,7 +132,6 @@ internal void CreateRules (SourceJavadocToXmldocGrammar grammar) }; CodeElementDeclaration.Rule = CodeElementContentTerm; - CodeElementDeclaration.Rule = CreateStartElement ("code", grammar) + InlineDeclarations + CreateEndElement ("code", grammar); CodeElementDeclaration.AstConfig.NodeCreator = (context, parseNode) => { // Parse the entire element captured in the token var codeElementText = parseNode.ChildNodes [0].Token.Text;