Support loading an OpenApiDocument with remote references.

darrelmiller · darrelmiller · commit 5090c01e8297 · 2021-03-02T21:49:23.000-05:00
diff --git a/src/Microsoft.OpenApi.Readers/OpenApiStreamReader.cs b/src/Microsoft.OpenApi.Readers/OpenApiStreamReader.cs
@@ -57,6 +57,7 @@ public async Task<ReadResult> ReadAsync(Stream input)
                 // YamlDocument doesn't support async reading.
                 bufferedStream = new MemoryStream();
                 await input.CopyToAsync(bufferedStream);
+                bufferedStream.Position = 0;
             }
 
             var reader = new StreamReader(bufferedStream);
diff --git a/src/Microsoft.OpenApi.Readers/OpenApiTextReaderReader.cs b/src/Microsoft.OpenApi.Readers/OpenApiTextReaderReader.cs
@@ -53,6 +53,11 @@ public OpenApiDocument Read(TextReader input, out OpenApiDiagnostic diagnostic)
             return new OpenApiYamlDocumentReader(this._settings).Read(yamlDocument, out diagnostic);
         }
 
+        /// <summary>
+        /// Reads the content of the TextReader.  If there are references to external documents then they will be read asynchronously.
+        /// </summary>
+        /// <param name="input">TextReader containing OpenAPI description to parse.</param>
+        /// <returns>A ReadResult instance that contains the resulting OpenApiDocument and a diagnostics instance.</returns>
         public async Task<ReadResult> ReadAsync(TextReader input)
         {
             YamlDocument yamlDocument;
diff --git a/src/Microsoft.OpenApi.Readers/OpenApiYamlDocumentReader.cs b/src/Microsoft.OpenApi.Readers/OpenApiYamlDocumentReader.cs
@@ -2,6 +2,7 @@
 // Licensed under the MIT license. 
 
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Threading.Tasks;
 using Microsoft.OpenApi.Exceptions;
@@ -134,28 +135,38 @@ private void ResolveReferences(OpenApiDiagnostic diagnostic, OpenApiDocument doc
 
         private async Task ResolveReferencesAsync(OpenApiDiagnostic diagnostic, OpenApiDocument document)
         {
+            List<OpenApiError> errors = new List<OpenApiError>();
+
             // Resolve References if requested
             switch (_settings.ReferenceResolution)
             {
                 case ReferenceResolutionSetting.ResolveAllReferences:
+
+                    // Create workspace for all documents to live in.
                     var openApiWorkSpace = new OpenApiWorkspace();
-                    document.Workspace = openApiWorkSpace;
-                    var streamLoader = new DefaultStreamLoader();
 
+                    // Load this root document into the workspace
+                    var streamLoader = new DefaultStreamLoader();
                     var workspaceLoader = new OpenApiWorkspaceLoader(openApiWorkSpace, _settings.CustomExternalLoader ?? streamLoader, _settings);
                     await workspaceLoader.LoadAsync(new OpenApiReference() { ExternalResource = "/" }, document);
-                    break;
-                case ReferenceResolutionSetting.ResolveLocalReferences:
-                    var errors = document.ResolveReferences(false);
 
-                    foreach (var item in errors)
+                    // Resolve all references in all the documents loaded into the OpenApiWorkspace
+                    foreach (var doc in openApiWorkSpace.Documents)
                     {
-                        diagnostic.Errors.Add(item);
+                        errors.AddRange(doc.ResolveReferences(true));
                     }
                     break;
+                case ReferenceResolutionSetting.ResolveLocalReferences:
+                    errors.AddRange(document.ResolveReferences(false));
+                    break;
                 case ReferenceResolutionSetting.DoNotResolveReferences:
                     break;
             }
+
+            foreach (var item in errors)
+            {
+                diagnostic.Errors.Add(item);
+            }
         }
 
 
diff --git a/src/Microsoft.OpenApi.Readers/Services/OpenApiRemoteReferenceCollector.cs b/src/Microsoft.OpenApi.Readers/Services/OpenApiRemoteReferenceCollector.cs
@@ -2,6 +2,7 @@
 // Licensed under the MIT license. 
 
 using System.Collections.Generic;
+using Microsoft.OpenApi.Interfaces;
 using Microsoft.OpenApi.Models;
 using Microsoft.OpenApi.Services;
 
@@ -19,9 +20,6 @@ public OpenApiRemoteReferenceCollector(OpenApiDocument document)
             _document = document;
         }
 
-        // TODO PathItem
-        // TODO Example
-
         /// <summary>
         /// List of external references collected from OpenApiDocument
         /// </summary>
@@ -33,60 +31,12 @@ public IEnumerable<OpenApiReference> References
         }
 
         /// <summary>
-        /// Collect external reference
-        /// </summary>
-        public override void Visit(OpenApiParameter parameter)
-        {
-            AddReference(parameter.Reference);
-        }
-
-        /// <summary>
-        /// Collect external reference
-        /// </summary>
-        public override void Visit(OpenApiCallback callback)
-        {
-            AddReference(callback.Reference);
-
-        }
-
-        /// <summary>
-        /// Collect external reference
-        /// </summary>
-        public override void Visit(OpenApiLink link)
-        {
-            AddReference(link.Reference);
-        }
-
-        /// <summary>
-        /// Collect external reference
-        /// </summary>
-        public override void Visit(OpenApiRequestBody requestBody)
-        {
-            AddReference(requestBody.Reference);
-        }
-
-        /// <summary>
-        /// Collect external reference
-        /// </summary>
-        public override void Visit(OpenApiResponse header)
-        {
-            AddReference(header.Reference);
-        }
-
-        /// <summary>
-        /// Collect external reference
-        /// </summary>
-        public override void Visit(OpenApiHeader header)
-        {
-            AddReference(header.Reference);
-        }
-
-        /// <summary>
-        /// Collect external reference
+        /// Collect reference for each reference 
         /// </summary>
-        public override void Visit(OpenApiSchema schema)
+        /// <param name="referenceable"></param>
+        public override void Visit(IOpenApiReferenceable referenceable)
         {
-             AddReference(schema.Reference);
+            AddReference(referenceable.Reference);
         }
 
         /// <summary>
@@ -104,7 +54,6 @@ private void AddReference(OpenApiReference reference)
                     }
                 }
             }
-        }
-    
+        }    
     }
 }
diff --git a/src/Microsoft.OpenApi.Readers/Services/OpenApiWorkspaceLoader.cs b/src/Microsoft.OpenApi.Readers/Services/OpenApiWorkspaceLoader.cs
@@ -43,7 +43,7 @@ internal async Task LoadAsync(OpenApiReference reference, OpenApiDocument docume
                 // If not already in workspace, load it and process references
                 if (!_workspace.Contains(item.ExternalResource))
                 {
-                    var input = _loader.Load(new Uri(item.ExternalResource));
+                    var input = await _loader.LoadAsync(new Uri(item.ExternalResource, UriKind.RelativeOrAbsolute));
                     var result = await reader.ReadAsync(input); // TODO merge _diagnositics
                     await LoadAsync(item, result.OpenApiDocument);
                 }
diff --git a/src/Microsoft.OpenApi.Readers/V2/OpenApiV2VersionService.cs b/src/Microsoft.OpenApi.Readers/V2/OpenApiV2VersionService.cs
@@ -161,6 +161,7 @@ public OpenApiReference ConvertToOpenApiReference(string reference, ReferenceTyp
                     return new OpenApiReference
                     {
                         ExternalResource = segments[0],
+                        Type = type,
                         Id = segments[1].Substring(1)
                     };
                 }
diff --git a/src/Microsoft.OpenApi.Readers/V3/OpenApiV3VersionService.cs b/src/Microsoft.OpenApi.Readers/V3/OpenApiV3VersionService.cs
@@ -69,6 +69,7 @@ public OpenApiReference ConvertToOpenApiReference(
                         // "$ref": "Pet.json"
                         return new OpenApiReference
                         {
+                            Type = type,
                             ExternalResource = segments[0]
                         };
                     }
@@ -89,12 +90,18 @@ public OpenApiReference ConvertToOpenApiReference(
                         // "$ref": "#/components/schemas/Pet"
                         return ParseLocalReference(segments[1]);
                     }
-
+                    // Where fragments point into a non-OpenAPI document, the id will be the complete fragment identifier
+                    string id = segments[1];
                     // $ref: externalSource.yaml#/Pet
+                    if (id.StartsWith("/components/"))
+                    {
+                        id = segments[1].Split('/')[3];
+                    } 
                     return new OpenApiReference
                     {
                         ExternalResource = segments[0],
-                        Id = segments[1].Substring(1)
+                        Type = type,
+                        Id = id
                     };
                 }
             }
diff --git a/src/Microsoft.OpenApi/Models/OpenApiDocument.cs b/src/Microsoft.OpenApi/Models/OpenApiDocument.cs
@@ -348,11 +348,15 @@ public IOpenApiReferenceable ResolveReference(OpenApiReference reference, bool u
                 return null;
             }
 
-            if (reference.IsExternal && !useExternal)
+            // Todo: Verify if we need to check to see if this external reference is actually targeted at this document.
+            if (useExternal)
             {
-                // Should not attempt to resolve external references against a single document.
-                throw new ArgumentException(Properties.SRResource.RemoteReferenceNotSupported);
-            }
+                if (this.Workspace == null)
+                {
+                    throw new ArgumentException(Properties.SRResource.WorkspaceRequredForExternalReferenceResolution);
+                }
+                return this.Workspace.ResolveReference(reference);
+            } 
 
             if (!reference.Type.HasValue)
             {
diff --git a/src/Microsoft.OpenApi/Properties/SRResource.Designer.cs b/src/Microsoft.OpenApi/Properties/SRResource.Designer.cs
diff --git a/src/Microsoft.OpenApi/Properties/SRResource.resx b/src/Microsoft.OpenApi/Properties/SRResource.resx
@@ -216,4 +216,7 @@
   <data name="Validation_StringMustBeEmailAddress" xml:space="preserve">
     <value>The string '{0}' MUST be in the format of an email address.</value>
   </data>
+  <data name="WorkspaceRequredForExternalReferenceResolution" xml:space="preserve">
+    <value>OpenAPI document must be added to an OpenApiWorkspace to be able to resolve external references.</value>
+  </data>
 </root>
diff --git a/src/Microsoft.OpenApi/Services/OpenApiWorkspace.cs b/src/Microsoft.OpenApi/Services/OpenApiWorkspace.cs
@@ -117,7 +117,7 @@ public IOpenApiReferenceable ResolveReference(OpenApiReference reference)
         {
             if (_documents.TryGetValue(new Uri(BaseUrl, reference.ExternalResource), out var doc))
             {
-                return doc.ResolveReference(reference, true);
+                return doc.ResolveReference(reference, false);
             }
             else if (_fragments.TryGetValue(new Uri(BaseUrl, reference.ExternalResource), out var fragment))
             {
diff --git a/test/Microsoft.OpenApi.Readers.Tests/Microsoft.OpenApi.Readers.Tests.csproj b/test/Microsoft.OpenApi.Readers.Tests/Microsoft.OpenApi.Readers.Tests.csproj
@@ -14,6 +14,8 @@
     </PropertyGroup>
     <ItemGroup>
       <None Remove="V2Tests\Samples\ComponentRootReference.json" />
+      <None Remove="V3Tests\Samples\OpenApiWorkspace\TodoComponents.yaml" />
+      <None Remove="V3Tests\Samples\OpenApiWorkspace\TodoMain.yaml" />
     </ItemGroup>
     <ItemGroup>
       <EmbeddedResource Include="OpenApiReaderTests\Samples\unsupported.v1.yaml">
@@ -232,6 +234,8 @@
       <EmbeddedResource Include="V3Tests\Samples\OpenApiSecurityScheme\openIdConnectSecurityScheme.yaml">
         <CopyToOutputDirectory>Never</CopyToOutputDirectory>
       </EmbeddedResource>
+      <EmbeddedResource Include="V3Tests\Samples\OpenApiWorkspace\TodoComponents.yaml" />
+      <EmbeddedResource Include="V3Tests\Samples\OpenApiWorkspace\TodoMain.yaml" />
       <EmbeddedResource Include="V3Tests\Samples\OpenApiXml\basicXml.yaml">
         <CopyToOutputDirectory>Never</CopyToOutputDirectory>
       </EmbeddedResource>
diff --git a/test/Microsoft.OpenApi.Readers.Tests/OpenApiWorkspaceTests/OpenApiWorkspaceStreamTests.cs b/test/Microsoft.OpenApi.Readers.Tests/OpenApiWorkspaceTests/OpenApiWorkspaceStreamTests.cs
@@ -14,7 +14,7 @@ namespace Microsoft.OpenApi.Readers.Tests.OpenApiWorkspaceTests
 {
     public class OpenApiWorkspaceStreamTests
     {
-
+        private const string SampleFolderPath = "V3Tests/Samples/OpenApiWorkspace/";
 
         // Use OpenApiWorkspace to load a document and a referenced document
 
@@ -45,6 +45,44 @@ public async Task LoadDocumentIntoWorkspace()
             Assert.NotNull(result.OpenApiDocument.Workspace);
 
         }
+
+
+        [Fact]
+        public async Task LoadTodoDocumentIntoWorkspace()
+        {
+            // Create a reader that will resolve all references
+            var reader = new OpenApiStreamReader(new OpenApiReaderSettings()
+            {
+                ReferenceResolution = ReferenceResolutionSetting.ResolveAllReferences,
+                CustomExternalLoader = new ResourceLoader()
+            });
+
+            ReadResult result;
+            using (var stream = Resources.GetStream("V3Tests/Samples/OpenApiWorkspace/TodoMain.yaml"))
+            {
+                result = await reader.ReadAsync(stream);
+            }
+
+            Assert.NotNull(result.OpenApiDocument.Workspace);
+            Assert.True(result.OpenApiDocument.Workspace.Contains("TodoComponents.yaml"));
+            var referencedSchema = result.OpenApiDocument
+                                            .Paths["/todos"]
+                                            .Operations[OperationType.Get]
+                                            .Responses["200"]
+                                            .Content["application/json"]
+                                                .Schema;
+            Assert.Equal("object", referencedSchema.Type);
+            Assert.Equal("string", referencedSchema.Properties["subject"].Type);
+            Assert.False(referencedSchema.UnresolvedReference);
+
+            var referencedParameter = result.OpenApiDocument
+                                            .Paths["/todos"]
+                                            .Operations[OperationType.Get]
+                                            .Parameters
+                                            .Where(p => p.Name == "filter").FirstOrDefault();
+            Assert.Equal("string", referencedParameter.Schema.Type);
+
+        }
     }
 
     public class MockLoader : IStreamLoader
@@ -59,4 +97,20 @@ public async Task<Stream> LoadAsync(Uri uri)
             return null;
         }
     }
+    
+
+    public class ResourceLoader : IStreamLoader
+    {
+        public Stream Load(Uri uri)
+        {
+            return null;
+        }
+
+        public async Task<Stream> LoadAsync(Uri uri)
+        {
+            var path = new Uri(new Uri("http://example.org/V3Tests/Samples/OpenApiWorkspace/"), uri).AbsolutePath;
+            path = path.Substring(1); // remove leading slash
+            return Resources.GetStream(path);
+        }
+    }
 }
diff --git a/test/Microsoft.OpenApi.Readers.Tests/ReferenceService/ConvertToOpenApiReferenceV3Tests.cs b/test/Microsoft.OpenApi.Readers.Tests/ReferenceService/ConvertToOpenApiReferenceV3Tests.cs
@@ -17,8 +17,8 @@ public void ParseExternalReference()
             // Arrange
             var versionService = new OpenApiV3VersionService();
             var externalResource = "externalSchema.json";
-            var id = "externalPathSegment1/externalPathSegment2/externalPathSegment3";
-            var input = $"{externalResource}#/{id}";
+            var id = "/externalPathSegment1/externalPathSegment2/externalPathSegment3";
+            var input = $"{externalResource}#{id}";
 
             // Act
             var reference = versionService.ConvertToOpenApiReference(input, null);
diff --git a/test/Microsoft.OpenApi.Readers.Tests/V3Tests/Samples/OpenApiWorkspace/TodoComponents.yaml b/test/Microsoft.OpenApi.Readers.Tests/V3Tests/Samples/OpenApiWorkspace/TodoComponents.yaml
diff --git a/test/Microsoft.OpenApi.Readers.Tests/V3Tests/Samples/OpenApiWorkspace/TodoMain.yaml b/test/Microsoft.OpenApi.Readers.Tests/V3Tests/Samples/OpenApiWorkspace/TodoMain.yaml
diff --git a/test/Microsoft.OpenApi.SmokeTests/WorkspaceTests.cs b/test/Microsoft.OpenApi.SmokeTests/WorkspaceTests.cs

Original file line number	Diff line number	Diff line change
`@@ -57,6 +57,7 @@ public async Task<ReadResult> ReadAsync(Stream input)`
`57`	`57`	`// YamlDocument doesn't support async reading.`
`58`	`58`	`bufferedStream = new MemoryStream();`
`59`	`59`	`await input.CopyToAsync(bufferedStream);`
	`60`	`+ bufferedStream.Position = 0;`
`60`	`61`	`}`
`61`	`62`
`62`	`63`	`var reader = new StreamReader(bufferedStream);`
Original file line number	Diff line number	Diff line change
`@@ -53,6 +53,11 @@ public OpenApiDocument Read(TextReader input, out OpenApiDiagnostic diagnostic)`
`53`	`53`	`return new OpenApiYamlDocumentReader(this._settings).Read(yamlDocument, out diagnostic);`
`54`	`54`	`}`
`55`	`55`
	`56`	`+ /// <summary>`
	`57`	`+ /// Reads the content of the TextReader. If there are references to external documents then they will be read asynchronously.`
	`58`	`+ /// </summary>`
	`59`	`+ /// <param name="input">TextReader containing OpenAPI description to parse.</param>`
	`60`	`+ /// <returns>A ReadResult instance that contains the resulting OpenApiDocument and a diagnostics instance.</returns>`
`56`	`61`	`public async Task<ReadResult> ReadAsync(TextReader input)`
`57`	`62`	`{`
`58`	`63`	`YamlDocument yamlDocument;`
Original file line number	Diff line number	Diff line change
`@@ -43,7 +43,7 @@ internal async Task LoadAsync(OpenApiReference reference, OpenApiDocument docume`
`43`	`43`	`// If not already in workspace, load it and process references`
`44`	`44`	`if (!_workspace.Contains(item.ExternalResource))`
`45`	`45`	`{`
`46`		`- var input = _loader.Load(new Uri(item.ExternalResource));`
	`46`	`+ var input = await _loader.LoadAsync(new Uri(item.ExternalResource, UriKind.RelativeOrAbsolute));`
`47`	`47`	`var result = await reader.ReadAsync(input); // TODO merge _diagnositics`
`48`	`48`	`await LoadAsync(item, result.OpenApiDocument);`
`49`	`49`	`}`
Original file line number	Diff line number	Diff line change
`@@ -161,6 +161,7 @@ public OpenApiReference ConvertToOpenApiReference(string reference, ReferenceTyp`
`161`	`161`	`return new OpenApiReference`
`162`	`162`	`{`
`163`	`163`	`ExternalResource = segments[0],`
	`164`	`+ Type = type,`
`164`	`165`	`Id = segments[1].Substring(1)`
`165`	`166`	`};`
`166`	`167`	`}`
Original file line number	Diff line number	Diff line change
`@@ -69,6 +69,7 @@ public OpenApiReference ConvertToOpenApiReference(`
`69`	`69`	`// "$ref": "Pet.json"`
`70`	`70`	`return new OpenApiReference`
`71`	`71`	`{`
	`72`	`+ Type = type,`
`72`	`73`	`ExternalResource = segments[0]`
`73`	`74`	`};`
`74`	`75`	`}`
`@@ -89,12 +90,18 @@ public OpenApiReference ConvertToOpenApiReference(`
`89`	`90`	`// "$ref": "#/components/schemas/Pet"`
`90`	`91`	`return ParseLocalReference(segments[1]);`
`91`	`92`	`}`
`92`		`-`
	`93`	`+ // Where fragments point into a non-OpenAPI document, the id will be the complete fragment identifier`
	`94`	`+ string id = segments[1];`
`93`	`95`	`// $ref: externalSource.yaml#/Pet`
	`96`	`+ if (id.StartsWith("/components/"))`
	`97`	`+ {`
	`98`	`+ id = segments[1].Split('/')[3];`
	`99`	`+ }`
`94`	`100`	`return new OpenApiReference`
`95`	`101`	`{`
`96`	`102`	`ExternalResource = segments[0],`
`97`		`- Id = segments[1].Substring(1)`
	`103`	`+ Type = type,`
	`104`	`+ Id = id`
`98`	`105`	`};`
`99`	`106`	`}`
`100`	`107`	`}`
Original file line number	Diff line number	Diff line change
`@@ -348,11 +348,15 @@ public IOpenApiReferenceable ResolveReference(OpenApiReference reference, bool u`
`348`	`348`	`return null;`
`349`	`349`	`}`
`350`	`350`
`351`		`- if (reference.IsExternal && !useExternal)`
	`351`	`+ // Todo: Verify if we need to check to see if this external reference is actually targeted at this document.`
	`352`	`+ if (useExternal)`
`352`	`353`	`{`
`353`		`- // Should not attempt to resolve external references against a single document.`
`354`		`- throw new ArgumentException(Properties.SRResource.RemoteReferenceNotSupported);`
`355`		`- }`
	`354`	`+ if (this.Workspace == null)`
	`355`	`+ {`
	`356`	`+ throw new ArgumentException(Properties.SRResource.WorkspaceRequredForExternalReferenceResolution);`
	`357`	`+ }`
	`358`	`+ return this.Workspace.ResolveReference(reference);`
	`359`	`+ }`
`356`	`360`
`357`	`361`	`if (!reference.Type.HasValue)`
`358`	`362`	`{`