diff --git a/src/Microsoft.OpenApi.Hidi/OpenApiService.cs b/src/Microsoft.OpenApi.Hidi/OpenApiService.cs index 2fdcd23aa..c15f77d6f 100644 --- a/src/Microsoft.OpenApi.Hidi/OpenApiService.cs +++ b/src/Microsoft.OpenApi.Hidi/OpenApiService.cs @@ -29,7 +29,10 @@ namespace Microsoft.OpenApi.Hidi { public class OpenApiService { - public static async Task ProcessOpenApiDocument( + /// + /// Implementation of the transform command + /// + public static async Task TransformOpenApiDocument( string openapi, string csdl, FileInfo output, @@ -74,126 +77,210 @@ CancellationToken cancellationToken if (!string.IsNullOrEmpty(csdl)) { - // Default to yaml and OpenApiVersion 3 during csdl to OpenApi conversion - openApiFormat = format ?? GetOpenApiFormat(csdl, logger); - openApiVersion = version == null ? OpenApiSpecVersion.OpenApi3_0 : TryParseOpenApiSpecVersion(version); - - stream = await GetStream(csdl, logger, cancellationToken); - document = await ConvertCsdlToOpenApi(stream); + using (logger.BeginScope($"Convert CSDL: {csdl}", csdl)) + { + stopwatch.Start(); + // Default to yaml and OpenApiVersion 3 during csdl to OpenApi conversion + openApiFormat = format ?? GetOpenApiFormat(csdl, logger); + openApiVersion = version != null ? TryParseOpenApiSpecVersion(version) : OpenApiSpecVersion.OpenApi3_0; + + stream = await GetStream(csdl, logger, cancellationToken); + document = await ConvertCsdlToOpenApi(stream); + stopwatch.Stop(); + logger.LogTrace("{timestamp}ms: Generated OpenAPI with {paths} paths.", stopwatch.ElapsedMilliseconds, document.Paths.Count); + } } else { stream = await GetStream(openapi, logger, cancellationToken); - // Parsing OpenAPI file - stopwatch.Start(); - logger.LogTrace("Parsing OpenApi file"); - var result = await new OpenApiStreamReader(new OpenApiReaderSettings + using (logger.BeginScope($"Parse OpenAPI: {openapi}",openapi)) { - ReferenceResolution = resolveexternal ? ReferenceResolutionSetting.ResolveAllReferences : ReferenceResolutionSetting.ResolveLocalReferences, - RuleSet = ValidationRuleSet.GetDefaultRuleSet() - } - ).ReadAsync(stream); + stopwatch.Restart(); + var result = await new OpenApiStreamReader(new OpenApiReaderSettings + { + ReferenceResolution = resolveexternal ? ReferenceResolutionSetting.ResolveAllReferences : ReferenceResolutionSetting.ResolveLocalReferences, + RuleSet = ValidationRuleSet.GetDefaultRuleSet() + } + ).ReadAsync(stream); - document = result.OpenApiDocument; - stopwatch.Stop(); + document = result.OpenApiDocument; - var context = result.OpenApiDiagnostic; - if (context.Errors.Count > 0) - { - logger.LogTrace("{timestamp}ms: Parsed OpenAPI with errors. {count} paths found.", stopwatch.ElapsedMilliseconds, document.Paths.Count); + var context = result.OpenApiDiagnostic; + if (context.Errors.Count > 0) + { + logger.LogTrace("{timestamp}ms: Parsed OpenAPI with errors. {count} paths found.", stopwatch.ElapsedMilliseconds, document.Paths.Count); - var errorReport = new StringBuilder(); + var errorReport = new StringBuilder(); - foreach (var error in context.Errors) + foreach (var error in context.Errors) + { + logger.LogError("OpenApi Parsing error: {message}", error.ToString()); + errorReport.AppendLine(error.ToString()); + } + logger.LogError($"{stopwatch.ElapsedMilliseconds}ms: OpenApi Parsing errors {string.Join(Environment.NewLine, context.Errors.Select(e => e.Message).ToArray())}"); + } + else { - logger.LogError("OpenApi Parsing error: {message}", error.ToString()); - errorReport.AppendLine(error.ToString()); + logger.LogTrace("{timestamp}ms: Parsed OpenApi successfully. {count} paths found.", stopwatch.ElapsedMilliseconds, document.Paths.Count); } - logger.LogError($"{stopwatch.ElapsedMilliseconds}ms: OpenApi Parsing errors {string.Join(Environment.NewLine, context.Errors.Select(e => e.Message).ToArray())}"); + + openApiFormat = format ?? GetOpenApiFormat(openapi, logger); + openApiVersion = version != null ? TryParseOpenApiSpecVersion(version) : result.OpenApiDiagnostic.SpecificationVersion; + stopwatch.Stop(); } - else + } + + using (logger.BeginScope("Filter")) + { + Func predicate = null; + + // Check if filter options are provided, then slice the OpenAPI document + if (!string.IsNullOrEmpty(filterbyoperationids) && !string.IsNullOrEmpty(filterbytags)) { - logger.LogTrace("{timestamp}ms: Parsed OpenApi successfully. {count} paths found.", stopwatch.ElapsedMilliseconds, document.Paths.Count); + throw new InvalidOperationException("Cannot filter by operationIds and tags at the same time."); } + if (!string.IsNullOrEmpty(filterbyoperationids)) + { + logger.LogTrace("Creating predicate based on the operationIds supplied."); + predicate = OpenApiFilterService.CreatePredicate(operationIds: filterbyoperationids); - openApiFormat = format ?? GetOpenApiFormat(openapi, logger); - openApiVersion = version == null ? TryParseOpenApiSpecVersion(version) : result.OpenApiDiagnostic.SpecificationVersion; - } + } + if (!string.IsNullOrEmpty(filterbytags)) + { + logger.LogTrace("Creating predicate based on the tags supplied."); + predicate = OpenApiFilterService.CreatePredicate(tags: filterbytags); - Func predicate; + } + if (!string.IsNullOrEmpty(filterbycollection)) + { + var fileStream = await GetStream(filterbycollection, logger, cancellationToken); + var requestUrls = ParseJsonCollectionFile(fileStream, logger); - // Check if filter options are provided, then slice the OpenAPI document - if (!string.IsNullOrEmpty(filterbyoperationids) && !string.IsNullOrEmpty(filterbytags)) - { - throw new InvalidOperationException("Cannot filter by operationIds and tags at the same time."); + logger.LogTrace("Creating predicate based on the paths and Http methods defined in the Postman collection."); + predicate = OpenApiFilterService.CreatePredicate(requestUrls: requestUrls, source: document); + } + if (predicate != null) + { + stopwatch.Restart(); + document = OpenApiFilterService.CreateFilteredDocument(document, predicate); + stopwatch.Stop(); + logger.LogTrace("{timestamp}ms: Creating filtered OpenApi document with {paths} paths.", stopwatch.ElapsedMilliseconds, document.Paths.Count); + } } - if (!string.IsNullOrEmpty(filterbyoperationids)) - { - logger.LogTrace("Creating predicate based on the operationIds supplied."); - predicate = OpenApiFilterService.CreatePredicate(operationIds: filterbyoperationids); - logger.LogTrace("Creating subset OpenApi document."); - document = OpenApiFilterService.CreateFilteredDocument(document, predicate); - } - if (!string.IsNullOrEmpty(filterbytags)) + using (logger.BeginScope("Output")) { - logger.LogTrace("Creating predicate based on the tags supplied."); - predicate = OpenApiFilterService.CreatePredicate(tags: filterbytags); + ; + using var outputStream = output?.Create(); + var textWriter = outputStream != null ? new StreamWriter(outputStream) : Console.Out; - logger.LogTrace("Creating subset OpenApi document."); - document = OpenApiFilterService.CreateFilteredDocument(document, predicate); - } - if (!string.IsNullOrEmpty(filterbycollection)) - { - var fileStream = await GetStream(filterbycollection, logger, cancellationToken); - var requestUrls = ParseJsonCollectionFile(fileStream, logger); + var settings = new OpenApiWriterSettings() + { + ReferenceInline = inline ? ReferenceInlineSetting.InlineLocalReferences : ReferenceInlineSetting.DoNotInlineReferences + }; + + IOpenApiWriter writer = openApiFormat switch + { + OpenApiFormat.Json => new OpenApiJsonWriter(textWriter, settings), + OpenApiFormat.Yaml => new OpenApiYamlWriter(textWriter, settings), + _ => throw new ArgumentException("Unknown format"), + }; - logger.LogTrace("Creating predicate based on the paths and Http methods defined in the Postman collection."); - predicate = OpenApiFilterService.CreatePredicate(requestUrls: requestUrls, source: document); + logger.LogTrace("Serializing to OpenApi document using the provided spec version and writer"); - logger.LogTrace("Creating subset OpenApi document."); - document = OpenApiFilterService.CreateFilteredDocument(document, predicate); + stopwatch.Start(); + document.Serialize(writer, openApiVersion); + stopwatch.Stop(); + + logger.LogTrace($"Finished serializing in {stopwatch.ElapsedMilliseconds}ms"); + textWriter.Flush(); } + return 0; + } + catch (Exception ex) + { +#if DEBUG + logger.LogCritical(ex, ex.Message); +#else + logger.LogCritical(ex.Message); + +#endif + return 1; + } + } - logger.LogTrace("Creating a new file"); - using var outputStream = output?.Create(); - var textWriter = outputStream != null ? new StreamWriter(outputStream) : Console.Out; + /// + /// Implementation of the validate command + /// + public static async Task ValidateOpenApiDocument( + string openapi, + LogLevel loglevel, + CancellationToken cancellationToken) + { + var logger = ConfigureLoggerInstance(loglevel); - var settings = new OpenApiWriterSettings() + try + { + if (string.IsNullOrEmpty(openapi)) { - ReferenceInline = inline ? ReferenceInlineSetting.InlineLocalReferences : ReferenceInlineSetting.DoNotInlineReferences - }; + throw new ArgumentNullException(nameof(openapi)); + } + var stream = await GetStream(openapi, logger, cancellationToken); - IOpenApiWriter writer = openApiFormat switch + OpenApiDocument document; + Stopwatch stopwatch = Stopwatch.StartNew(); + using (logger.BeginScope($"Parsing OpenAPI: {openapi}", openapi)) { - OpenApiFormat.Json => new OpenApiJsonWriter(textWriter, settings), - OpenApiFormat.Yaml => new OpenApiYamlWriter(textWriter, settings), - _ => throw new ArgumentException("Unknown format"), - }; + stopwatch.Start(); - logger.LogTrace("Serializing to OpenApi document using the provided spec version and writer"); + var result = await new OpenApiStreamReader(new OpenApiReaderSettings + { + RuleSet = ValidationRuleSet.GetDefaultRuleSet() + } + ).ReadAsync(stream); - stopwatch.Start(); - document.Serialize(writer, openApiVersion); - stopwatch.Stop(); + logger.LogTrace("{timestamp}ms: Completed parsing.", stopwatch.ElapsedMilliseconds); + + document = result.OpenApiDocument; + var context = result.OpenApiDiagnostic; + if (context.Errors.Count != 0) + { + using (logger.BeginScope("Detected errors")) + { + foreach (var error in context.Errors) + { + logger.LogError(error.ToString()); + } + } + } + stopwatch.Stop(); + } - logger.LogTrace($"Finished serializing in {stopwatch.ElapsedMilliseconds}ms"); - textWriter.Flush(); + using (logger.BeginScope("Calculating statistics")) + { + var statsVisitor = new StatsVisitor(); + var walker = new OpenApiWalker(statsVisitor); + walker.Walk(document); + + logger.LogTrace("Finished walking through the OpenApi document. Generating a statistics report.."); + logger.LogInformation(statsVisitor.GetStatisticsReport()); + } return 0; } catch (Exception ex) { -#if DEBUG +#if DEBUG logger.LogCritical(ex, ex.Message); #else logger.LogCritical(ex.Message); #endif return 1; - } + } + } - + /// /// Converts CSDL to OpenAPI /// @@ -229,71 +316,6 @@ public static async Task ConvertCsdlToOpenApi(Stream csdl) return document; } - /// - /// Fixes the references in the resulting OpenApiDocument. - /// - /// The converted OpenApiDocument. - /// A valid OpenApiDocument instance. - public static OpenApiDocument FixReferences(OpenApiDocument document) - { - // This method is only needed because the output of ConvertToOpenApi isn't quite a valid OpenApiDocument instance. - // So we write it out, and read it back in again to fix it up. - - var sb = new StringBuilder(); - document.SerializeAsV3(new OpenApiYamlWriter(new StringWriter(sb))); - var doc = new OpenApiStringReader().Read(sb.ToString(), out _); - - return doc; - } - - private static async Task GetStream(string input, ILogger logger, CancellationToken cancellationToken) - { - var stopwatch = new Stopwatch(); - stopwatch.Start(); - - Stream stream; - if (input.StartsWith("http")) - { - try - { - var httpClientHandler = new HttpClientHandler() - { - SslProtocols = System.Security.Authentication.SslProtocols.Tls12, - }; - using var httpClient = new HttpClient(httpClientHandler) - { - DefaultRequestVersion = HttpVersion.Version20 - }; - stream = await httpClient.GetStreamAsync(input, cancellationToken); - } - catch (HttpRequestException ex) - { - throw new InvalidOperationException($"Could not download the file at {input}", ex); - } - } - else - { - try - { - var fileInput = new FileInfo(input); - stream = fileInput.OpenRead(); - } - catch (Exception ex) when (ex is FileNotFoundException || - ex is PathTooLongException || - ex is DirectoryNotFoundException || - ex is IOException || - ex is UnauthorizedAccessException || - ex is SecurityException || - ex is NotSupportedException) - { - throw new InvalidOperationException($"Could not open the file at {input}", ex); - } - } - stopwatch.Stop(); - logger.LogTrace("{timestamp}ms: Read file {input}", stopwatch.ElapsedMilliseconds, input); - return stream; - } - /// /// Takes in a file stream, parses the stream into a JsonDocument and gets a list of paths and Http methods /// @@ -326,55 +348,83 @@ public static Dictionary> ParseJsonCollectionFile(Stream st return requestUrls; } - internal static async Task ValidateOpenApiDocument(string openapi, LogLevel loglevel, CancellationToken cancellationToken) + /// + /// Fixes the references in the resulting OpenApiDocument. + /// + /// The converted OpenApiDocument. + /// A valid OpenApiDocument instance. + private static OpenApiDocument FixReferences(OpenApiDocument document) { - var logger = ConfigureLoggerInstance(loglevel); + // This method is only needed because the output of ConvertToOpenApi isn't quite a valid OpenApiDocument instance. + // So we write it out, and read it back in again to fix it up. - try + var sb = new StringBuilder(); + document.SerializeAsV3(new OpenApiYamlWriter(new StringWriter(sb))); + var doc = new OpenApiStringReader().Read(sb.ToString(), out _); + + return doc; + } + + /// + /// Reads stream from file system or makes HTTP request depending on the input string + /// + private static async Task GetStream(string input, ILogger logger, CancellationToken cancellationToken) + { + Stream stream; + using (logger.BeginScope("Reading input stream")) { - if (string.IsNullOrEmpty(openapi)) - { - throw new ArgumentNullException(nameof(openapi)); - } - var stream = await GetStream(openapi, logger, cancellationToken); + var stopwatch = new Stopwatch(); + stopwatch.Start(); - OpenApiDocument document; - logger.LogTrace("Parsing the OpenApi file"); - document = new OpenApiStreamReader(new OpenApiReaderSettings + if (input.StartsWith("http")) { - RuleSet = ValidationRuleSet.GetDefaultRuleSet() + try + { + var httpClientHandler = new HttpClientHandler() + { + SslProtocols = System.Security.Authentication.SslProtocols.Tls12, + }; + using var httpClient = new HttpClient(httpClientHandler) + { + DefaultRequestVersion = HttpVersion.Version20 + }; + stream = await httpClient.GetStreamAsync(input, cancellationToken); + } + catch (HttpRequestException ex) + { + throw new InvalidOperationException($"Could not download the file at {input}", ex); + } } - ).Read(stream, out var context); - - if (context.Errors.Count != 0) + else { - foreach (var error in context.Errors) + try { - logger.LogError("OpenApi Parsing error: {message}", error.ToString()); + var fileInput = new FileInfo(input); + stream = fileInput.OpenRead(); + } + catch (Exception ex) when (ex is FileNotFoundException || + ex is PathTooLongException || + ex is DirectoryNotFoundException || + ex is IOException || + ex is UnauthorizedAccessException || + ex is SecurityException || + ex is NotSupportedException) + { + throw new InvalidOperationException($"Could not open the file at {input}", ex); } } - - var statsVisitor = new StatsVisitor(); - var walker = new OpenApiWalker(statsVisitor); - walker.Walk(document); - - logger.LogTrace("Finished walking through the OpenApi document. Generating a statistics report.."); - logger.LogInformation(statsVisitor.GetStatisticsReport()); - - return 0; - } - catch(Exception ex) - { -#if DEBUG - logger.LogCritical(ex, ex.Message); -#else - logger.LogCritical(ex.Message); -#endif - return 1; + stopwatch.Stop(); + logger.LogTrace("{timestamp}ms: Read file {input}", stopwatch.ElapsedMilliseconds, input); } - + return stream; } + /// + /// Attempt to guess OpenAPI format based in input URL + /// + /// + /// + /// private static OpenApiFormat GetOpenApiFormat(string input, ILogger logger) { logger.LogTrace("Getting the OpenApi format"); @@ -390,8 +440,10 @@ private static ILogger ConfigureLoggerInstance(LogLevel loglevel) var logger = LoggerFactory.Create((builder) => { builder - .AddConsole() -#if DEBUG + .AddSimpleConsole(c => { + c.IncludeScopes = true; + }) +#if DEBUG .AddDebug() #endif .SetMinimumLevel(loglevel); diff --git a/src/Microsoft.OpenApi.Hidi/Program.cs b/src/Microsoft.OpenApi.Hidi/Program.cs index c83d8fd17..ac39aaad4 100644 --- a/src/Microsoft.OpenApi.Hidi/Program.cs +++ b/src/Microsoft.OpenApi.Hidi/Program.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. +using System; using System.CommandLine; using System.IO; using System.Threading; @@ -11,7 +12,7 @@ namespace Microsoft.OpenApi.Hidi { static class Program { - static async Task Main(string[] args) + static async Task Main(string[] args) { var rootCommand = new RootCommand() { }; @@ -35,7 +36,7 @@ static async Task Main(string[] args) var formatOption = new Option("--format", "File format"); formatOption.AddAlias("-f"); - var logLevelOption = new Option("--loglevel", () => LogLevel.Warning, "The log level to use when logging messages to the main output."); + var logLevelOption = new Option("--loglevel", () => LogLevel.Information, "The log level to use when logging messages to the main output."); logLevelOption.AddAlias("-ll"); var filterByOperationIdsOption = new Option("--filter-by-operationids", "Filters OpenApiDocument by OperationId(s) provided"); @@ -78,13 +79,16 @@ static async Task Main(string[] args) }; transformCommand.SetHandler ( - OpenApiService.ProcessOpenApiDocument, descriptionOption, csdlOption, outputOption, versionOption, formatOption, logLevelOption, inlineOption, resolveExternalOption, filterByOperationIdsOption, filterByTagsOption, filterByCollectionOption); + OpenApiService.TransformOpenApiDocument, descriptionOption, csdlOption, outputOption, cleanOutputOption, versionOption, formatOption, logLevelOption, inlineOption, resolveExternalOption, filterByOperationIdsOption, filterByTagsOption, filterByCollectionOption); rootCommand.Add(transformCommand); rootCommand.Add(validateCommand); // Parse the incoming args and invoke the handler - return await rootCommand.InvokeAsync(args); + await rootCommand.InvokeAsync(args); + + //// Wait for logger to write messages to the console before exiting + await Task.Delay(10); } } } diff --git a/src/Microsoft.OpenApi.Hidi/appsettings.json b/src/Microsoft.OpenApi.Hidi/appsettings.json deleted file mode 100644 index 882248cf8..000000000 --- a/src/Microsoft.OpenApi.Hidi/appsettings.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "Logging": { - "LogLevel": { - "Default": "Debug" - } - } -} \ No newline at end of file