build: fix docs-content missing some inherited member description (#24310)

This happened due to: 9973f1d

Recently we added logic to exclude resolved external API docs from the Dgeni
pipeline, preventing them from being processed by the Dgeni JSDoc processor.

This was necessary due to a bug in Dgeni which resulted in runtime errors for
unknown JSDoc tags. Some tags from external API docs were unknown because
the framework repository uses different sets of annotations/tags.

At this point, it seemed reasonable to never add such external API docs to the
Dgeni pipeline, while also working around this runtime exception that way.

Now after seeing the results, it became clear that we still want to continue
processing external API docs as some of our public-facing API docs might end
up merging external API document members.

In this docs-content built it can be observed that some inherited/merged
members no longer have a proper resolved description:

angular/material2-docs-content@6b602cf

This commit fixes it by adding these external JSDoc tags temporarily as known tags until
the Dgeni upstream fix is available. As part of this change we introduced a custom processor
to error for unknown JSDoc tags (we want to be strict about this). This will become helpful
as Dgeni only warns about unknown tags (and this is non-configurable). Also while being at it
the types for `tags` has been improved to avoid the use of `any` everywhere.

(cherry picked from commit ebc3e10)

Author: devversion

tools/dgeni/common/decorators.ts

@@ -3,19 +3,20 @@ import {ClassExportDoc} from 'dgeni-packages/typescript/api-doc-types/ClassExpor
 import {MemberDoc} from 'dgeni-packages/typescript/api-doc-types/MemberDoc';
 import {PropertyMemberDoc} from 'dgeni-packages/typescript/api-doc-types/PropertyMemberDoc';
 import {CategorizedClassDoc, DeprecationInfo, HasDecoratorsDoc} from './dgeni-definitions';
+import {findJsDocTag, hasJsDocTag} from './tags';
 
-export function isMethod(doc: MemberDoc) {
+export function isMethod(doc: MemberDoc): boolean {
   return doc.hasOwnProperty('parameters') && !doc.isGetAccessor && !doc.isSetAccessor;
 }
 
-export function isGenericTypeParameter(doc: MemberDoc) {
+export function isGenericTypeParameter(doc: MemberDoc): boolean {
   if (doc.containerDoc instanceof ClassExportDoc) {
-    return doc.containerDoc.typeParams && `<${doc.name}>` === doc.containerDoc.typeParams;
+    return !!doc.containerDoc.typeParams && `<${doc.name}>` === doc.containerDoc.typeParams;
   }
   return false;
 }
 
-export function isProperty(doc: MemberDoc) {
+export function isProperty(doc: MemberDoc): boolean {
   if (
     doc instanceof PropertyMemberDoc ||
     // The latest Dgeni version no longer treats getters or setters as properties.
@@ -28,30 +29,28 @@ export function isProperty(doc: MemberDoc) {
   return false;
 }
 
-export function isDirective(doc: ClassExportDoc) {
+export function isDirective(doc: ClassExportDoc): boolean {
   return hasClassDecorator(doc, 'Component') || hasClassDecorator(doc, 'Directive');
 }
 
-export function isService(doc: ClassExportDoc) {
+export function isService(doc: ClassExportDoc): boolean {
   return hasClassDecorator(doc, 'Injectable');
 }
 
-export function isNgModule(doc: ClassExportDoc) {
+export function isNgModule(doc: ClassExportDoc): boolean {
   return hasClassDecorator(doc, 'NgModule');
 }
 
-export function isDeprecatedDoc(doc: any) {
-  return ((doc.tags && doc.tags.tags) || []).some((tag: any) => tag.tagName === 'deprecated');
+export function isDeprecatedDoc(doc: ApiDoc): boolean {
+  return hasJsDocTag(doc, 'deprecated');
 }
 
 /** Whether the given document is annotated with the "@docs-primary-export" jsdoc tag. */
-export function isPrimaryExportDoc(doc: any) {
-  return ((doc.tags && doc.tags.tags) || []).some(
-    (tag: any) => tag.tagName === 'docs-primary-export',
-  );
+export function isPrimaryExportDoc(doc: ApiDoc): boolean {
+  return hasJsDocTag(doc, 'docs-primary-export');
 }
 
-export function getDirectiveSelectors(classDoc: CategorizedClassDoc) {
+export function getDirectiveSelectors(classDoc: CategorizedClassDoc): string[] | undefined {
   if (classDoc.directiveMetadata) {
     const directiveSelectors: string = classDoc.directiveMetadata.get('selector');
 
@@ -65,28 +64,24 @@ export function getDirectiveSelectors(classDoc: CategorizedClassDoc) {
   return undefined;
 }
 
-export function hasMemberDecorator(doc: MemberDoc, decoratorName: string) {
+export function hasMemberDecorator(doc: MemberDoc, decoratorName: string): boolean {
   return doc.docType == 'member' && hasDecorator(doc, decoratorName);
 }
 
-export function hasClassDecorator(doc: ClassExportDoc, decoratorName: string) {
+export function hasClassDecorator(doc: ClassExportDoc, decoratorName: string): boolean {
   return doc.docType == 'class' && hasDecorator(doc, decoratorName);
 }
 
-export function hasDecorator(doc: HasDecoratorsDoc, decoratorName: string) {
+export function hasDecorator(doc: HasDecoratorsDoc, decoratorName: string): boolean {
   return (
     !!doc.decorators &&
     doc.decorators.length > 0 &&
     doc.decorators.some(d => d.name == decoratorName)
   );
 }
 
-export function getBreakingChange(doc: any): string | null {
-  if (!doc.tags) {
-    return null;
-  }
-
-  const breakingChange = doc.tags.tags.find((t: any) => t.tagName === 'breaking-change');
+export function getBreakingChange(doc: ApiDoc): string | null {
+  const breakingChange = findJsDocTag(doc, 'breaking-change');
   return breakingChange ? breakingChange.description : null;
 }
 

tools/dgeni/common/private-docs.ts

@@ -1,6 +1,7 @@
 import {ApiDoc} from 'dgeni-packages/typescript/api-doc-types/ApiDoc';
 import {MemberDoc} from 'dgeni-packages/typescript/api-doc-types/MemberDoc';
 import {isInheritanceCreatedDoc} from './class-inheritance';
+import {findJsDocTag, hasJsDocTag, Tag} from './tags';
 
 const INTERNAL_METHODS = [
   // Lifecycle methods
@@ -48,20 +49,18 @@ export function isPublicDoc(doc: ApiDoc) {
 }
 
 /** Gets the @docs-public tag from the given document if present. */
-export function getDocsPublicTag(doc: any): {tagName: string; description: string} | undefined {
-  const tags = doc.tags && doc.tags.tags;
-  return tags ? tags.find((d: any) => d.tagName == 'docs-public') : undefined;
+export function getDocsPublicTag(doc: ApiDoc): Tag | undefined {
+  return findJsDocTag(doc, 'docs-public');
 }
 
 /** Whether the given method member is listed as an internal member. */
-function _isInternalMember(memberDoc: MemberDoc) {
+function _isInternalMember(memberDoc: MemberDoc): boolean {
   return INTERNAL_METHODS.includes(memberDoc.name);
 }
 
 /** Whether the given doc has a @docs-private tag set. */
-function _hasDocsPrivateTag(doc: any) {
-  const tags = doc.tags && doc.tags.tags;
-  return tags ? tags.find((d: any) => d.tagName == 'docs-private') : false;
+function _hasDocsPrivateTag(doc: ApiDoc): boolean {
+  return hasJsDocTag(doc, 'docs-private');
 }
 
 /**
@@ -73,6 +72,6 @@ function _hasDocsPrivateTag(doc: any) {
  * split up into several base classes to support the MDC prototypes. e.g. "_MatMenu" should
  * show up in the docs as "MatMenu".
  */
-function _isEnforcedPublicDoc(doc: any): boolean {
+function _isEnforcedPublicDoc(doc: ApiDoc): boolean {
   return getDocsPublicTag(doc) !== undefined;
 }

tools/dgeni/common/tags.ts

@@ -0,0 +1,72 @@
+import {ApiDoc} from 'dgeni-packages/typescript/api-doc-types/ApiDoc';
+
+/**
+ * Type describing a collection of tags, matching with the objects
+ * created by the Dgeni JSDoc processors.
+ *
+ * https://github.com/angular/dgeni-packages/blob/19e629c0d156572cbea149af9e0cc7ec02db7cb6/jsdoc/lib/TagCollection.js#L4
+ */
+export interface TagCollection {
+  /** List of tags. */
+  tags: Tag[];
+  /** Map which maps tag names to their tag instances. */
+  tagsByName: Map<string, Tag[]>;
+  /** List of tags which are unkown, or have errors. */
+  badTags: Tag[];
+}
+
+/**
+ * Type describing a tag, matching with the objects created by the
+ * Dgeni JSDoc processors.
+ *
+ * https://github.com/angular/dgeni-packages/blob/19e629c0d156572cbea149af9e0cc7ec02db7cb6/jsdoc/lib/Tag.js#L1
+ */
+export interface Tag {
+  /** Definition of the tag. Undefined if the tag is unknown. */
+  tagDef: undefined | TagDefinition;
+  /** Name of the tag (excluding the `@`) */
+  tagName: string;
+  /** Description associated with the tag. */
+  description: string;
+  /** Source file line where this tag starts. */
+  startingLine: number;
+  /** Optional list of errors that have been computed for this tag. */
+  errors?: string[];
+}
+
+/** Type describing a tag definition for the Dgeni JSDoc processor. */
+export interface TagDefinition {
+  /** Name of the tag (excluding the `@`) */
+  name: string;
+  /** Property where the tag information should be attached to. */
+  docProperty?: string;
+  /** Whether multiple instances of the tag can be used in the same comment. */
+  multi?: boolean;
+  /** Whether this tag is required for all API documents. */
+  required?: boolean;
+}
+
+/** Type describing an API doc with JSDoc tag information. */
+export type ApiDocWithJsdocTags = ApiDoc & {
+  /** Collection of JSDoc tags attached to this API document. */
+  tags: TagCollection;
+};
+
+/** Whether the specified API document has JSDoc tag information attached. */
+export function isApiDocWithJsdocTags(doc: ApiDoc): doc is ApiDocWithJsdocTags {
+  return (doc as Partial<ApiDocWithJsdocTags>).tags !== undefined;
+}
+
+/** Finds the specified JSDoc tag within the given API doc. */
+export function findJsDocTag(doc: ApiDoc, tagName: string): Tag | undefined {
+  if (!isApiDocWithJsdocTags(doc)) {
+    return undefined;
+  }
+
+  return doc.tags.tags.find(t => t.tagName === tagName);
+}
+
+/** Gets whether the specified API doc has a given JSDoc tag. */
+export function hasJsDocTag(doc: ApiDoc, tagName: string): boolean {
+  return findJsDocTag(doc, tagName) !== undefined;
+}

tools/dgeni/docs-package.ts

@@ -8,6 +8,7 @@ import {AsyncFunctionsProcessor} from './processors/async-functions';
 import {categorizer} from './processors/categorizer';
 import {DocsPrivateFilter} from './processors/docs-private-filter';
 import {EntryPointGrouper} from './processors/entry-point-grouper';
+import {ErrorUnknownJsdocTagsProcessor} from './processors/error-unknown-jsdoc-tags';
 import {FilterDuplicateExports} from './processors/filter-duplicate-exports';
 import {mergeInheritedProperties} from './processors/merge-inherited-properties';
 import {resolveInheritedDocs} from './processors/resolve-inherited-docs';
@@ -51,6 +52,9 @@ apiDocsPackage.processor(mergeInheritedProperties);
 // Processor that filters out symbols that should not be shown in the docs.
 apiDocsPackage.processor(new DocsPrivateFilter());
 
+// Processor that throws an error if API docs with unknown JSDoc tags are discovered.
+apiDocsPackage.processor(new ErrorUnknownJsdocTagsProcessor());
+
 // Processor that appends categorization flags to the docs, e.g. `isDirective`, `isNgModule`, etc.
 apiDocsPackage.processor(categorizer);
 
@@ -100,6 +104,15 @@ apiDocsPackage.config(function (parseTagsProcessor: any) {
     {name: 'template', multi: true},
     //  JSDoc annotations/tags which are not supported by default.
     {name: 'throws', multi: true},
+
+    // Annotations/tags from external API docs (i.e. from the node modules). These tags are
+    // added so that no errors are reported.
+    // TODO(devversion): remove this once the fix in dgeni-package is available.
+    //   https://github.com/angular/dgeni-packages/commit/19e629c0d156572cbea149af9e0cc7ec02db7cb6.
+    {name: 'usageNotes'},
+    {name: 'publicApi'},
+    {name: 'ngModule', multi: true},
+    {name: 'nodoc'},
   ]);
 });
 

tools/dgeni/processors/error-unknown-jsdoc-tags.ts

@@ -0,0 +1,38 @@
+import {DocCollection, Processor} from 'dgeni';
+import {isApiDocWithJsdocTags} from '../common/tags';
+
+/**
+ * Processor that checks API docs for unknown JSDoc tags. Dgeni by default will
+ * warn about unknown tags. This processor will throw an error instead.
+ */
+export class ErrorUnknownJsdocTagsProcessor implements Processor {
+  name = 'error-unknown-tags';
+  $runAfter = ['docs-private-filter'];
+  $runBefore = ['categorizer'];
+
+  $process(docs: DocCollection) {
+    for (const doc of docs) {
+      if (!isApiDocWithJsdocTags(doc)) {
+        continue;
+      }
+
+      if (doc.tags.badTags.length > 0) {
+        let errorMessage = `Found errors for processed JSDoc comments in ${doc.id}:\n`;
+
+        for (const tag of doc.tags.badTags) {
+          errorMessage += '\n';
+
+          if (tag.tagDef === undefined) {
+            errorMessage += `  * Tag "${tag.tagName}": Unknown tag.\n`;
+          }
+
+          for (const concreteError of tag.errors ?? []) {
+            errorMessage += `  * Tag "${tag.tagName}": ${concreteError}\n`;
+          }
+        }
+
+        throw new Error(errorMessage);
+      }
+    }
+  }
+}

tools/dgeni/processors/resolve-inherited-docs.ts

@@ -20,6 +20,9 @@ export function resolveInheritedDocs(exportSymbolsToDocsMap: Map<ts.Symbol, Clas
  * processed by other standard processors in the Dgeni pipeline. This is helpful as
  * API documents for inheritance are created manually if not exported, and we'd want
  * such docs to be processed by the Dgeni JSDoc processor for example.
+ *
+ * Note that we also want to include external API docs (e.g. from the node modules)
+ * since members from those can also be merged with public-facing API docs.
  */
 export class ResolveInheritedDocs implements Processor {
   $runBefore = ['docs-private-filter', 'parsing-tags'];
@@ -44,14 +47,6 @@ export class ResolveInheritedDocs implements Processor {
         if (!isInheritanceCreatedDoc(apiDoc)) {
           return;
         }
-        // If this is an external document not part of our sources, we will not add it to the
-        // Dgeni API doc pipeline. Docs would be filtered regardless, but adding them to the
-        // pipeline could cause e.g. the JSDoc parser to complain about tags/annotations which
-        // are not known/relevant to our API docs. e.g. Framework uses `@nodoc` or `@usageNotes`.
-        if (apiDoc.fileInfo.projectRelativePath.startsWith('../')) {
-          return;
-        }
-
         // Add the member docs for the inherited doc to the Dgeni doc collection.
         this._getContainingMemberDocs(apiDoc).forEach(d => newDocs.add(d));
         // Add the class-like export doc to the Dgeni doc collection.