From bf3edad137d3539724c51c1daa5064b29d5a87d2 Mon Sep 17 00:00:00 2001 From: Anatoli Beliaev Date: Tue, 20 Aug 2019 21:12:02 -0700 Subject: [PATCH 1/2] Handle paginated PSGallery response --- .../IPowerShellGallerySearchInvoker.cs | 15 + .../PowerShellGalleryModuleProvider.cs | 92 ++--- .../PowerShellGallerySearchInvoker.cs | 43 +++ .../PowerShellGalleryModuleProviderTests.cs | 347 ++++++++++++++++++ 4 files changed, 454 insertions(+), 43 deletions(-) create mode 100644 src/DependencyManagement/IPowerShellGallerySearchInvoker.cs create mode 100644 src/DependencyManagement/PowerShellGallerySearchInvoker.cs create mode 100644 test/Unit/DependencyManagement/PowerShellGalleryModuleProviderTests.cs diff --git a/src/DependencyManagement/IPowerShellGallerySearchInvoker.cs b/src/DependencyManagement/IPowerShellGallerySearchInvoker.cs new file mode 100644 index 00000000..62e3743b --- /dev/null +++ b/src/DependencyManagement/IPowerShellGallerySearchInvoker.cs @@ -0,0 +1,15 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +namespace Microsoft.Azure.Functions.PowerShellWorker.DependencyManagement +{ + using System; + using System.IO; + + internal interface IPowerShellGallerySearchInvoker + { + Stream Invoke(Uri uri); + } +} diff --git a/src/DependencyManagement/PowerShellGalleryModuleProvider.cs b/src/DependencyManagement/PowerShellGalleryModuleProvider.cs index 3b38e0b7..feb581d6 100644 --- a/src/DependencyManagement/PowerShellGalleryModuleProvider.cs +++ b/src/DependencyManagement/PowerShellGalleryModuleProvider.cs @@ -6,9 +6,7 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.DependencyManagement { using System; - using System.IO; using System.Management.Automation; - using System.Net.Http; using System.Xml; using Microsoft.Azure.Functions.PowerShellWorker.PowerShell; @@ -16,6 +14,13 @@ namespace Microsoft.Azure.Functions.PowerShellWorker.DependencyManagement internal class PowerShellGalleryModuleProvider : IModuleProvider { + private readonly IPowerShellGallerySearchInvoker _searchInvoker; + + public PowerShellGalleryModuleProvider(IPowerShellGallerySearchInvoker searchInvoker = null) + { + _searchInvoker = searchInvoker ?? new PowerShellGallerySearchInvoker(); + } + /// /// Returns the latest module version from the PSGallery for the given module name and major version. /// @@ -27,38 +32,18 @@ public string GetLatestPublishedModuleVersion(string moduleName, string majorVer Uri address = new Uri($"{PowerShellGalleryFindPackagesByIdUri}'{moduleName}'"); - string latestMajorVersion = null; - Stream stream = null; + var expectedVersionStart = majorVersion + "."; + + Version latestVersion = null; - var retryCount = 3; - while (true) + do { - using (var client = new HttpClient()) + var stream = _searchInvoker.Invoke(address); + if (stream == null) { - try - { - var response = client.GetAsync(address).Result; - - // Throw is not a successful request - response.EnsureSuccessStatusCode(); - - stream = response.Content.ReadAsStreamAsync().Result; - break; - } - catch (Exception) - { - if (retryCount <= 0) - { - throw; - } - - retryCount--; - } + break; } - } - if (stream != null) - { // Load up the XML response XmlDocument doc = new XmlDocument(); using (XmlReader reader = XmlReader.Create(stream)) @@ -68,27 +53,20 @@ public string GetLatestPublishedModuleVersion(string moduleName, string majorVer // Add the namespaces for the gallery xml content XmlNamespaceManager nsmgr = new XmlNamespaceManager(doc.NameTable); - nsmgr.AddNamespace("ps", "http://www.w3.org/2005/Atom"); + nsmgr.AddNamespace("a", "http://www.w3.org/2005/Atom"); nsmgr.AddNamespace("d", "http://schemas.microsoft.com/ado/2007/08/dataservices"); nsmgr.AddNamespace("m", "http://schemas.microsoft.com/ado/2007/08/dataservices/metadata"); - // Find the version information XmlNode root = doc.DocumentElement; - var props = root.SelectNodes("//m:properties[d:IsPrerelease = \"false\"]/d:Version", nsmgr); + latestVersion = GetLatestVersion(root, nsmgr, expectedVersionStart, latestVersion); - if (props != null && props.Count > 0) - { - foreach (XmlNode prop in props) - { - if (prop.FirstChild.Value.StartsWith(majorVersion)) - { - latestMajorVersion = prop.FirstChild.Value; - } - } - } + // The response may be paginated. In this case, the current page + // contains a link to the next page. + address = GetNextLink(root, nsmgr); } + while (address != null); - return latestMajorVersion; + return latestVersion?.ToString(); } /// @@ -125,5 +103,33 @@ public void Cleanup(PowerShell pwsh) .AddParameter("ErrorAction", "SilentlyContinue") .InvokeAndClearCommands(); } + + private static Version GetLatestVersion( + XmlNode root, XmlNamespaceManager namespaceManager, string expectedVersionStart, Version latestVersion) + { + var versions = root.SelectNodes("/a:feed/a:entry/m:properties[d:IsPrerelease = \"false\"]/d:Version", namespaceManager); + if (versions != null) + { + foreach (XmlNode prop in versions) + { + if (prop.FirstChild.Value.StartsWith(expectedVersionStart) + && Version.TryParse(prop.FirstChild.Value, out var thisVersion)) + { + if (latestVersion == null || thisVersion > latestVersion) + { + latestVersion = thisVersion; + } + } + } + } + + return latestVersion; + } + + private static Uri GetNextLink(XmlNode root, XmlNamespaceManager namespaceManager) + { + var nextLink = root.SelectNodes("/a:feed/a:link[@rel=\"next\"]/@href", namespaceManager); + return nextLink.Count == 1 ? new Uri(nextLink[0].Value) : null; + } } } diff --git a/src/DependencyManagement/PowerShellGallerySearchInvoker.cs b/src/DependencyManagement/PowerShellGallerySearchInvoker.cs new file mode 100644 index 00000000..07e85aa8 --- /dev/null +++ b/src/DependencyManagement/PowerShellGallerySearchInvoker.cs @@ -0,0 +1,43 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. +// + +namespace Microsoft.Azure.Functions.PowerShellWorker.DependencyManagement +{ + using System; + using System.IO; + using System.Net.Http; + + internal class PowerShellGallerySearchInvoker : IPowerShellGallerySearchInvoker + { + public Stream Invoke(Uri uri) + { + var retryCount = 3; + while (true) + { + using (var client = new HttpClient()) + { + try + { + var response = client.GetAsync(uri).Result; + + // Throw is not a successful request + response.EnsureSuccessStatusCode(); + + return response.Content.ReadAsStreamAsync().Result; + } + catch (Exception) + { + if (retryCount <= 0) + { + throw; + } + + retryCount--; + } + } + } + } + } +} diff --git a/test/Unit/DependencyManagement/PowerShellGalleryModuleProviderTests.cs b/test/Unit/DependencyManagement/PowerShellGalleryModuleProviderTests.cs new file mode 100644 index 00000000..40637b4b --- /dev/null +++ b/test/Unit/DependencyManagement/PowerShellGalleryModuleProviderTests.cs @@ -0,0 +1,347 @@ +namespace Microsoft.Azure.Functions.PowerShellWorker.Test.DependencyManagement +{ + using System; + using System.IO; + using System.Text; + using Moq; + using Xunit; + + using PowerShellWorker.DependencyManagement; + + public class PowerShellGalleryModuleProviderTests + { + private readonly Mock _mockSearchInvoker = + new Mock(MockBehavior.Strict); + + [Fact] + public void ReturnsNullIfSearchInvokerReturnsNull() + { + _mockSearchInvoker.Setup(_ => _.Invoke(It.IsAny())).Returns(default(Stream)); + var moduleProvider = new PowerShellGalleryModuleProvider(_mockSearchInvoker.Object); + var actualVersion = moduleProvider.GetLatestPublishedModuleVersion("ModuleName", "1"); + Assert.Null(actualVersion); + } + + [Fact] + public void ReturnsSingleVersion() + { + const string ResponseText = @" + + + + + 1.2.3.4 + false + + +"; + + using (var responseStream = new MemoryStream(Encoding.UTF8.GetBytes(ResponseText))) + { + _mockSearchInvoker.Setup(_ => _.Invoke(It.IsAny())).Returns(responseStream); + var moduleProvider = new PowerShellGalleryModuleProvider(_mockSearchInvoker.Object); + var actualVersion = moduleProvider.GetLatestPublishedModuleVersion("ModuleName", "1"); + Assert.Equal("1.2.3.4", actualVersion); + } + } + + [Fact] + public void FindsLatestVersionRegardlessOfResponseOrder() + { + const string ResponseText = @" + + + + + 1.2.3.4 + false + + + + + 1.2.3.6 + false + + + + + 1.2.3.5 + false + + +"; + + using (var responseStream = new MemoryStream(Encoding.UTF8.GetBytes(ResponseText))) + { + _mockSearchInvoker.Setup(_ => _.Invoke(It.IsAny())).Returns(responseStream); + var moduleProvider = new PowerShellGalleryModuleProvider(_mockSearchInvoker.Object); + var actualVersion = moduleProvider.GetLatestPublishedModuleVersion("ModuleName", "1"); + Assert.Equal("1.2.3.6", actualVersion); + } + } + + [Fact] + public void IgnoresPrereleaseVersions() + { + const string ResponseText = @" + + + + + 1.2.3.4 + false + + + + + 1.2.3.6 + true + + + + + 1.2.3.5 + false + + +"; + + using (var responseStream = new MemoryStream(Encoding.UTF8.GetBytes(ResponseText))) + { + _mockSearchInvoker.Setup(_ => _.Invoke(It.IsAny())).Returns(responseStream); + var moduleProvider = new PowerShellGalleryModuleProvider(_mockSearchInvoker.Object); + var actualVersion = moduleProvider.GetLatestPublishedModuleVersion("ModuleName", "1"); + Assert.Equal("1.2.3.5", actualVersion); + } + } + + [Fact] + public void IgnoresNotMatchingMajorVersions() + { + const string ResponseText = @" + + + + + 0.2.3.7 + false + + + + + 1.2.3.5 + false + + + + + 1.2.3.6 + false + + + + + 2.2.3.7 + false + + +"; + + using (var responseStream = new MemoryStream(Encoding.UTF8.GetBytes(ResponseText))) + { + _mockSearchInvoker.Setup(_ => _.Invoke(It.IsAny())).Returns(responseStream); + var moduleProvider = new PowerShellGalleryModuleProvider(_mockSearchInvoker.Object); + var actualVersion = moduleProvider.GetLatestPublishedModuleVersion("ModuleName", "1"); + Assert.Equal("1.2.3.6", actualVersion); + } + } + + [Theory] + [InlineData("0.1", "0.2")] + [InlineData("0.1", "0.1.0")] + [InlineData("0.1", "0.1.1")] + [InlineData("0.1.2", "0.2.1")] + [InlineData("0.1.0", "0.1.0.1")] + [InlineData("0.1.2.3", "0.1.2.4")] + [InlineData("0.1.2.4", "0.2.2.3")] + public void ComparesVersionsCorrectly(string lowerVersion, string higherVersion) + { + const string ResponseTextTemplate = @" + + + + + {0} + false + + + + + {1} + false + + +"; + + var responseText = string.Format(ResponseTextTemplate, lowerVersion, higherVersion); + using (var responseStream = new MemoryStream(Encoding.UTF8.GetBytes(responseText))) + { + _mockSearchInvoker.Setup(_ => _.Invoke(It.IsAny())).Returns(responseStream); + var moduleProvider = new PowerShellGalleryModuleProvider(_mockSearchInvoker.Object); + var actualVersion = moduleProvider.GetLatestPublishedModuleVersion("ModuleName", "0"); + Assert.Equal(higherVersion, actualVersion); + } + } + + [Fact] + public void ReturnsNullIfNoVersionFound() + { + const string ResponseText = @" + + +"; + + using (var responseStream = new MemoryStream(Encoding.UTF8.GetBytes(ResponseText))) + { + _mockSearchInvoker.Setup(_ => _.Invoke(It.IsAny())).Returns(responseStream); + var moduleProvider = new PowerShellGalleryModuleProvider(_mockSearchInvoker.Object); + var actualVersion = moduleProvider.GetLatestPublishedModuleVersion("ModuleName", "1"); + Assert.Null(actualVersion); + } + } + + [Fact] + public void FindsLatestVersionAcrossMultiplePages() + { + const string ResponseText1 = @" + + + + + + 1.2.3.4 + false + + + + + 1.2.3.5 + false + + +"; + + const string ResponseText2 = @" + + + + + + 1.2.3.1 + false + + + + + 1.2.3.6 + false + + +"; + + const string ResponseText3 = @" + + + + + 1.2.3.2 + false + + + + + 1.2.3.3 + false + + +"; + + using (var responseStream1 = new MemoryStream(Encoding.UTF8.GetBytes(ResponseText1))) + using (var responseStream2 = new MemoryStream(Encoding.UTF8.GetBytes(ResponseText2))) + using (var responseStream3 = new MemoryStream(Encoding.UTF8.GetBytes(ResponseText3))) + { + _mockSearchInvoker.Setup(_ => _.Invoke(It.IsNotIn(new Uri("https://NextLink1"), new Uri("https://NextLink2")))) + .Returns(responseStream1); + + _mockSearchInvoker.Setup(_ => _.Invoke(new Uri("https://NextLink1"))) + .Returns(responseStream2); + + _mockSearchInvoker.Setup(_ => _.Invoke(new Uri("https://NextLink2"))) + .Returns(responseStream3); + + var moduleProvider = new PowerShellGalleryModuleProvider(_mockSearchInvoker.Object); + var actualVersion = moduleProvider.GetLatestPublishedModuleVersion("ModuleName", "1"); + Assert.Equal("1.2.3.6", actualVersion); + } + } + + [Theory] + [InlineData("Az", "0", "0.7.0")] + [InlineData("Az", "1", "1.8.0")] + [InlineData("Az", "2", "2.5.0")] + [InlineData("Az", "3", null)] + [InlineData("Az.Accounts", "0", null)] + [InlineData("Az.Accounts", "1", "1.6.1")] + [InlineData("Az.Accounts", "2", null)] + [InlineData("dbatools", "0", "0.9.834")] + [InlineData("dbatools", "1", "1.0.34")] + [InlineData("Pester", "0", null)] + [InlineData("Pester", "1", null)] + [InlineData("Pester", "2", null)] + [InlineData("Pester", "3", "3.4.6")] + [InlineData("Pester", "4", "4.8.1")] + [InlineData("Pester", "5", null)] + public void RetrievesLatestVersion(string moduleName, string majorVersion, string expectedVersion) + { + var moduleProvider = new PowerShellGalleryModuleProvider(); + var actualVersion = moduleProvider.GetLatestPublishedModuleVersion(moduleName, majorVersion); + Assert.Equal(expectedVersion, actualVersion); + } + } +} From 0ee7a96ba4c9b1cc70983f7ce85eb6b0b858681f Mon Sep 17 00:00:00 2001 From: Anatoli Beliaev Date: Tue, 20 Aug 2019 21:37:08 -0700 Subject: [PATCH 2/2] Remove RetrievesLatestVersion test --- .../PowerShellGalleryModuleProviderTests.cs | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/test/Unit/DependencyManagement/PowerShellGalleryModuleProviderTests.cs b/test/Unit/DependencyManagement/PowerShellGalleryModuleProviderTests.cs index 40637b4b..d72a0406 100644 --- a/test/Unit/DependencyManagement/PowerShellGalleryModuleProviderTests.cs +++ b/test/Unit/DependencyManagement/PowerShellGalleryModuleProviderTests.cs @@ -320,28 +320,5 @@ public void FindsLatestVersionAcrossMultiplePages() Assert.Equal("1.2.3.6", actualVersion); } } - - [Theory] - [InlineData("Az", "0", "0.7.0")] - [InlineData("Az", "1", "1.8.0")] - [InlineData("Az", "2", "2.5.0")] - [InlineData("Az", "3", null)] - [InlineData("Az.Accounts", "0", null)] - [InlineData("Az.Accounts", "1", "1.6.1")] - [InlineData("Az.Accounts", "2", null)] - [InlineData("dbatools", "0", "0.9.834")] - [InlineData("dbatools", "1", "1.0.34")] - [InlineData("Pester", "0", null)] - [InlineData("Pester", "1", null)] - [InlineData("Pester", "2", null)] - [InlineData("Pester", "3", "3.4.6")] - [InlineData("Pester", "4", "4.8.1")] - [InlineData("Pester", "5", null)] - public void RetrievesLatestVersion(string moduleName, string majorVersion, string expectedVersion) - { - var moduleProvider = new PowerShellGalleryModuleProvider(); - var actualVersion = moduleProvider.GetLatestPublishedModuleVersion(moduleName, majorVersion); - Assert.Equal(expectedVersion, actualVersion); - } } }