From 838766b7700b1e5835451247ef9b904a0e4069d5 Mon Sep 17 00:00:00 2001 From: Omotola Date: Tue, 1 Aug 2023 14:07:17 -0700 Subject: [PATCH] Users/oakeredolu/newpipservice (#683) Created new PythonResolver for SimplePypiClient --- .../pip/ISimplePythonResolver.cs | 16 + .../pip/PythonResolver.cs | 14 +- .../pip/PythonResolverState.cs | 16 + .../pip/SimplePythonResolver.cs | 373 ++++++++++++++++ .../Extensions/ServiceCollectionExtensions.cs | 1 + .../SimplePythonResolverTests.cs | 400 ++++++++++++++++++ 6 files changed, 807 insertions(+), 13 deletions(-) create mode 100644 src/Microsoft.ComponentDetection.Detectors/pip/ISimplePythonResolver.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/pip/PythonResolverState.cs create mode 100644 src/Microsoft.ComponentDetection.Detectors/pip/SimplePythonResolver.cs create mode 100644 test/Microsoft.ComponentDetection.Detectors.Tests/SimplePythonResolverTests.cs diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/ISimplePythonResolver.cs b/src/Microsoft.ComponentDetection.Detectors/pip/ISimplePythonResolver.cs new file mode 100644 index 000000000..321f7ea04 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/pip/ISimplePythonResolver.cs @@ -0,0 +1,16 @@ +namespace Microsoft.ComponentDetection.Detectors.Pip; + +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Contracts; + +public interface ISimplePythonResolver +{ + /// + /// Resolves the root Python packages from the initial list of packages. + /// + /// The component recorder for file that is been processed. + /// The initial list of packages. + /// The root packages, with dependencies associated as children. + Task> ResolveRootsAsync(ISingleFileComponentRecorder singleFileComponentRecorder, IList initialPackages); +} diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs index 823ebebaa..e0a4e078e 100644 --- a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolver.cs @@ -1,4 +1,4 @@ -namespace Microsoft.ComponentDetection.Detectors.Pip; +namespace Microsoft.ComponentDetection.Detectors.Pip; using System; using System.Collections.Generic; @@ -216,16 +216,4 @@ private void AddGraphNode(PythonResolverState state, PipGraphNode parent, string node.Parents.Add(parent); } } - - private class PythonResolverState - { - public IDictionary>> ValidVersionMap { get; } - = new Dictionary>>(StringComparer.OrdinalIgnoreCase); - - public Queue<(string PackageName, PipDependencySpecification Package)> ProcessingQueue { get; } = new Queue<(string, PipDependencySpecification)>(); - - public IDictionary NodeReferences { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); - - public IList Roots { get; } = new List(); - } } diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolverState.cs b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolverState.cs new file mode 100644 index 000000000..0f0ea5160 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/pip/PythonResolverState.cs @@ -0,0 +1,16 @@ +namespace Microsoft.ComponentDetection.Detectors.Pip; + +using System; +using System.Collections.Generic; + +public class PythonResolverState +{ + public IDictionary>> ValidVersionMap { get; } + = new Dictionary>>(StringComparer.OrdinalIgnoreCase); + + public Queue<(string PackageName, PipDependencySpecification Package)> ProcessingQueue { get; } = new Queue<(string, PipDependencySpecification)>(); + + public IDictionary NodeReferences { get; } = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public IList Roots { get; } = new List(); +} diff --git a/src/Microsoft.ComponentDetection.Detectors/pip/SimplePythonResolver.cs b/src/Microsoft.ComponentDetection.Detectors/pip/SimplePythonResolver.cs new file mode 100644 index 000000000..062218080 --- /dev/null +++ b/src/Microsoft.ComponentDetection.Detectors/pip/SimplePythonResolver.cs @@ -0,0 +1,373 @@ +namespace Microsoft.ComponentDetection.Detectors.Pip; + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +public class SimplePythonResolver : ISimplePythonResolver +{ + private readonly ISimplePyPiClient simplePypiClient; + private readonly ILogger logger; + + public SimplePythonResolver(ISimplePyPiClient simplePypiClient, ILogger logger) + { + this.simplePypiClient = simplePypiClient; + this.logger = logger; + } + + /// + public async Task> ResolveRootsAsync(ISingleFileComponentRecorder singleFileComponentRecorder, IList initialPackages) + { + var state = new PythonResolverState(); + + // Fill the dictionary with valid packages for the roots + foreach (var rootPackage in initialPackages) + { + // If we have it, we probably just want to skip at this phase as this indicates duplicates + if (!state.ValidVersionMap.TryGetValue(rootPackage.Name, out _)) + { + var simplePythonProject = await this.simplePypiClient.GetSimplePypiProjectAsync(rootPackage); + + if (simplePythonProject != null && simplePythonProject.Files.Any()) + { + var pythonProject = this.ConvertSimplePypiProjectToSortedDictionary(simplePythonProject, rootPackage); + + state.ValidVersionMap[rootPackage.Name] = pythonProject; + + // Grab the latest version as our candidate version + var candidateVersion = state.ValidVersionMap[rootPackage.Name].Keys.Any() + ? state.ValidVersionMap[rootPackage.Name].Keys.Last() : null; + + var node = new PipGraphNode(new PipComponent(rootPackage.Name, candidateVersion)); + + state.NodeReferences[rootPackage.Name] = node; + + state.Roots.Add(node); + + state.ProcessingQueue.Enqueue((rootPackage.Name, rootPackage)); + } + else + { + this.logger.LogWarning( + "Root dependency {RootPackageName} not found on pypi. Skipping package.", + rootPackage.Name); + singleFileComponentRecorder.RegisterPackageParseFailure(rootPackage.Name); + } + } + } + + // Now queue packages for processing + return await this.ProcessQueueAsync(singleFileComponentRecorder, state) ?? new List(); + } + + private async Task> ProcessQueueAsync(ISingleFileComponentRecorder singleFileComponentRecorder, PythonResolverState state) + { + while (state.ProcessingQueue.Count > 0) + { + var (root, currentNode) = state.ProcessingQueue.Dequeue(); + + // gather all dependencies for the current node + var dependencies = (await this.FetchPackageDependenciesAsync(state, currentNode)).Where(x => !x.PackageIsUnsafe()); + + foreach (var dependencyNode in dependencies) + { + // if we have already seen the dependency and the version we have is valid, just add the dependency to the graph + if (state.NodeReferences.TryGetValue(dependencyNode.Name, out var node) && + PythonVersionUtilities.VersionValidForSpec(node.Value.Version, dependencyNode.DependencySpecifiers)) + { + state.NodeReferences[currentNode.Name].Children.Add(node); + node.Parents.Add(state.NodeReferences[currentNode.Name]); + } + else if (node != null) + { + this.logger.LogWarning("Candidate version ({NodeValueId}) for {DependencyName} already exists in map and the version is NOT valid.", node.Value.Id, dependencyNode.Name); + this.logger.LogWarning("Specifiers: {DependencySpecifiers} for package {CurrentNodeName} caused this.", string.Join(',', dependencyNode.DependencySpecifiers), currentNode.Name); + + // The currently selected version is invalid, try to see if there is another valid version available + if (!await this.InvalidateAndReprocessAsync(state, node, dependencyNode)) + { + this.logger.LogWarning( + "Version Resolution for {DependencyName} failed, assuming last valid version is used.", + dependencyNode.Name); + + // there is no valid version available for the node, dependencies are incompatible, + } + } + else + { + // We haven't encountered this package before, so let's fetch it and find a candidate + var newProject = await this.simplePypiClient.GetSimplePypiProjectAsync(dependencyNode); + + if (newProject != null && newProject.Files.Any()) + { + var result = this.ConvertSimplePypiProjectToSortedDictionary(newProject, dependencyNode); + state.ValidVersionMap[dependencyNode.Name] = result; + var candidateVersion = state.ValidVersionMap[dependencyNode.Name].Keys.Any() + ? state.ValidVersionMap[dependencyNode.Name].Keys.Last() : null; + + this.AddGraphNode(state, state.NodeReferences[currentNode.Name], dependencyNode.Name, candidateVersion); + + state.ProcessingQueue.Enqueue((root, dependencyNode)); + } + else + { + this.logger.LogWarning( + "Dependency Package {DependencyName} not found in Pypi. Skipping package", + dependencyNode.Name); + singleFileComponentRecorder.RegisterPackageParseFailure(dependencyNode.Name); + } + } + } + } + + return state.Roots; + } + + /// + /// Converts a SimplePypiProject to a SortedDictionary of PythonProjectReleases. + /// + /// The SimplePypiProject gotten from the api. + /// The PipDependency Specification. + /// Returns a SortedDictionary of PythonProjectReleases. + private SortedDictionary> ConvertSimplePypiProjectToSortedDictionary(SimplePypiProject simplePypiProject, PipDependencySpecification spec) + { + var sortedProjectVersions = new SortedDictionary>(); + foreach (var file in simplePypiProject.Files) + { + try + { + var packageType = this.GetPackageType(file.FileName); + var version = this.GetVersionFromFileName(file.FileName); + var parsedVersion = PythonVersion.Create(version); + if (parsedVersion.Valid && parsedVersion.IsReleasedPackage && + PythonVersionUtilities.VersionValidForSpec(version, spec.DependencySpecifiers)) + { + var pythonProjectRelease = new PythonProjectRelease() { PythonVersion = version, PackageType = packageType, Size = file.Size, Url = file.Url }; + if (!sortedProjectVersions.ContainsKey(version)) + { + sortedProjectVersions.Add(version, new List()); + } + + sortedProjectVersions[version].Add(pythonProjectRelease); + } + } + catch (ArgumentException ae) + { + this.logger.LogError( + ae, + "Release {Release} could not be added to the sorted list of pip components for spec={SpecName}. Usually this happens with unexpected PyPi version formats (e.g. prerelease/dev versions).", + JsonConvert.SerializeObject(file), + spec.Name); + continue; + } + } + + return sortedProjectVersions; + } + + /// + /// Returns the package type based on the file name. + /// + /// the name of the file from simple pypi. + /// a string representing the package type. + private string GetPackageType(string fileName) + { + if (fileName.EndsWith(".whl")) + { + return "bdist_wheel"; + } + else if (fileName.EndsWith(".tar.gz")) + { + return "sdist"; + } + else if (fileName.EndsWith(".egg")) + { + return "bdist_egg"; + } + else + { + return string.Empty; + } + } + + /// + /// Uses regex to extract the version from the file name. + /// + /// the name of the file from simple pypi. + /// returns a string representing the release version. + private string GetVersionFromFileName(string fileName) + { + var version = Regex.Match(fileName, @"-(\d+\.\d+(\.\d)*)(.tar|-)").Groups[1]; + return version.Value; + } + + /// + /// Fetches the dependencies for a package. + /// + /// The PythonResolverState. + /// The PipDependencySpecification. + /// Returns a list of PipDependencySpecification. + private async Task> FetchPackageDependenciesAsync( + PythonResolverState state, + PipDependencySpecification spec) + { + var candidateVersion = state.NodeReferences[spec.Name].Value.Version; + + var packageToFetch = state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_wheel", x.PackageType, StringComparison.OrdinalIgnoreCase)) ?? + state.ValidVersionMap[spec.Name][candidateVersion].FirstOrDefault(x => string.Equals("bdist_egg", x.PackageType, StringComparison.OrdinalIgnoreCase)); + if (packageToFetch == null) + { + return new List(); + } + + var packageFileStream = await this.simplePypiClient.FetchPackageFileStreamAsync(packageToFetch.Url); + + if (packageFileStream.Length == 0) + { + return new List(); + } + + return await this.FetchDependenciesFromPackageStreamAsync(spec.Name, candidateVersion, packageFileStream); + } + + /// + /// Given a package stream will unzip and return the dependencies in the metadata file. + /// + /// The package name. + /// The package version. + /// The package file stream. + /// Returns a list of the dependencies. + private async Task> FetchDependenciesFromPackageStreamAsync(string name, string version, Stream packageStream) + { + var dependencies = new List(); + var package = new ZipArchive(packageStream); + + var entry = package.GetEntry($"{name.Replace('-', '_')}-{version}.dist-info/METADATA"); + + // If there is no metadata file, the package doesn't have any declared dependencies + if (entry == null) + { + return dependencies; + } + + var content = new List(); + using (var stream = entry.Open()) + { + using var streamReader = new StreamReader(stream); + + while (!streamReader.EndOfStream) + { + var line = await streamReader.ReadLineAsync(); + + if (PipDependencySpecification.RequiresDistRegex.IsMatch(line)) + { + content.Add(line); + } + } + } + + // Pull the packages that aren't conditional based on "extras" + // Right now we just want to resolve the graph as most comsumers will + // experience it + foreach (var deps in content.Where(x => !x.Contains("extra =="))) + { + dependencies.Add(new PipDependencySpecification(deps, true)); + } + + return dependencies; + } + + /// + /// Given a state, node, and new spec, will reprocess a new valid version for the node. + /// + /// The PythonResolverState. + /// The PipGraphNode. + /// The PipDependencySpecification. + /// Returns true if the node can be reprocessed else false. + private async Task InvalidateAndReprocessAsync( + PythonResolverState state, + PipGraphNode node, + PipDependencySpecification newSpec) + { + var pipComponent = node.Value; + + var oldVersions = state.ValidVersionMap[pipComponent.Name].Keys.ToList(); + var currentSelectedVersion = node.Value.Version; + var currentReleases = state.ValidVersionMap[pipComponent.Name][currentSelectedVersion]; + foreach (var version in oldVersions) + { + if (!PythonVersionUtilities.VersionValidForSpec(version, newSpec.DependencySpecifiers)) + { + state.ValidVersionMap[pipComponent.Name].Remove(version); + } + } + + if (state.ValidVersionMap[pipComponent.Name].Count == 0) + { + state.ValidVersionMap[pipComponent.Name][currentSelectedVersion] = currentReleases; + return false; + } + + var candidateVersion = state.ValidVersionMap[pipComponent.Name].Keys.Any() ? state.ValidVersionMap[pipComponent.Name].Keys.Last() : null; + + node.Value = new PipComponent(pipComponent.Name, candidateVersion); + + var dependencies = (await this.FetchPackageDependenciesAsync(state, newSpec)).ToDictionary(x => x.Name, x => x); + + var toRemove = new List(); + foreach (var child in node.Children) + { + var pipChild = child.Value; + + if (!dependencies.TryGetValue(pipChild.Name, out var newDependency)) + { + toRemove.Add(child); + } + else if (!PythonVersionUtilities.VersionValidForSpec(pipChild.Version, newDependency.DependencySpecifiers)) + { + if (!await this.InvalidateAndReprocessAsync(state, child, newDependency)) + { + return false; + } + } + } + + foreach (var remove in toRemove) + { + node.Children.Remove(remove); + } + + return true; + } + + /// + /// Adds a node to the graph. + /// + /// The PythonResolverState. + /// The parent node. + /// The package name. + /// The package version. + private void AddGraphNode(PythonResolverState state, PipGraphNode parent, string name, string version) + { + if (state.NodeReferences.TryGetValue(name, out var value)) + { + parent.Children.Add(value); + value.Parents.Add(parent); + } + else + { + var node = new PipGraphNode(new PipComponent(name, version)); + state.NodeReferences[name] = node; + parent.Children.Add(node); + node.Parents.Add(parent); + } + } +} diff --git a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs index 55ffc94e3..fb5ddeef2 100644 --- a/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs +++ b/src/Microsoft.ComponentDetection.Orchestrator/Extensions/ServiceCollectionExtensions.cs @@ -116,6 +116,7 @@ public static IServiceCollection AddComponentDetection(this IServiceCollection s services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); // pnpm diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/SimplePythonResolverTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/SimplePythonResolverTests.cs new file mode 100644 index 000000000..6428d5dd2 --- /dev/null +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/SimplePythonResolverTests.cs @@ -0,0 +1,400 @@ +namespace Microsoft.ComponentDetection.Detectors.Tests; + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Detectors.Pip; +using Microsoft.Extensions.Logging; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +[TestClass] +public class SimplePythonResolverTests +{ + private Mock> loggerMock; + private Mock simplePyPiClient; + private Mock recorderMock; + + [TestInitialize] + public void TestInitialize() + { + this.loggerMock = new Mock>(); + this.simplePyPiClient = new Mock(); + this.recorderMock = new Mock(); + } + + [TestMethod] + public async Task TestPipResolverSimpleGraphAsync() + { + var a = "a==1.0"; + var b = "b==1.0"; + var c = "c==1.0"; + + var specA = new PipDependencySpecification(a); + var specB = new PipDependencySpecification(b); + var specC = new PipDependencySpecification(c); + + var aReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.0", "bdist_wheel") }); + var bReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.0", "bdist_wheel") }); + var cReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.0", "bdist_wheel") }); + + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("a")))).ReturnsAsync(aReleases); + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("b")))).ReturnsAsync(bReleases); + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("c")))).ReturnsAsync(cReleases); + + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(aReleases.Files.First().Url)).ReturnsAsync(this.CreatePypiZip("a", "1.0", this.CreateMetadataString(new List() { b }))); + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(bReleases.Files.First().Url)).ReturnsAsync(this.CreatePypiZip("b", "1.0", this.CreateMetadataString(new List() { c }))); + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(cReleases.Files.First().Url)).ReturnsAsync(new MemoryStream()); + + var dependencies = new List { specA }; + + var resolver = new SimplePythonResolver(this.simplePyPiClient.Object, this.loggerMock.Object); + + var resolveResult = await resolver.ResolveRootsAsync(this.recorderMock.Object, dependencies); + + Assert.IsNotNull(resolveResult); + + var expectedA = new PipGraphNode(new PipComponent("a", "1.0")); + var expectedB = new PipGraphNode(new PipComponent("b", "1.0")); + var expectedC = new PipGraphNode(new PipComponent("c", "1.0")); + + expectedA.Children.Add(expectedB); + expectedB.Parents.Add(expectedA); + expectedB.Children.Add(expectedC); + expectedC.Parents.Add(expectedB); + + Assert.IsTrue(this.CompareGraphs(resolveResult.First(), expectedA)); + } + + [TestMethod] + public async Task TestPipResolverNonExistantRootAsync() + { + var a = "a==1.0"; + var b = "b==1.0"; + var c = "c==1.0"; + var doesNotExist = "dne==1.0"; + + var specA = new PipDependencySpecification(a); + var specB = new PipDependencySpecification(b); + var specC = new PipDependencySpecification(c); + var specDne = new PipDependencySpecification(doesNotExist); + + var aReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.0", "bdist_wheel") }); + var bReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.0", "bdist_wheel") }); + var cReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.0", "bdist_wheel") }); + + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("a")))).ReturnsAsync(aReleases); + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("b")))).ReturnsAsync(bReleases); + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("c")))).ReturnsAsync(cReleases); + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("dne")))).ReturnsAsync(this.CreateSimplePypiProject(new List<(string, string)>())); + + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(aReleases.Files.First().Url)).ReturnsAsync(this.CreatePypiZip("a", "1.0", this.CreateMetadataString(new List() { b }))); + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(bReleases.Files.First().Url)).ReturnsAsync(this.CreatePypiZip("b", "1.0", this.CreateMetadataString(new List() { c }))); + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(cReleases.Files.First().Url)).ReturnsAsync(new MemoryStream()); + + var dependencies = new List { specA, specDne }; + + var resolver = new SimplePythonResolver(this.simplePyPiClient.Object, this.loggerMock.Object); + + var resolveResult = await resolver.ResolveRootsAsync(this.recorderMock.Object, dependencies); + + Assert.IsNotNull(resolveResult); + + var expectedA = new PipGraphNode(new PipComponent("a", "1.0")); + var expectedB = new PipGraphNode(new PipComponent("b", "1.0")); + var expectedC = new PipGraphNode(new PipComponent("c", "1.0")); + + expectedA.Children.Add(expectedB); + expectedB.Parents.Add(expectedA); + expectedB.Children.Add(expectedC); + expectedC.Parents.Add(expectedB); + + Assert.IsTrue(this.CompareGraphs(resolveResult.First(), expectedA)); + } + + [TestMethod] + public async Task TestPipResolverNonExistantLeafAsync() + { + var a = "a==1.0"; + var b = "b==1.0"; + var c = "c==1.0"; + + var specA = new PipDependencySpecification(a); + var specB = new PipDependencySpecification(b); + var specC = new PipDependencySpecification(c); + var aReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.0", "bdist_wheel") }); + var bReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.0", "bdist_wheel") }); + var cReleases = this.CreateSimplePypiProject(new List<(string, string)> { }); + + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("a")))).ReturnsAsync(aReleases); + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("b")))).ReturnsAsync(bReleases); + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("c")))).ReturnsAsync(cReleases); + + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(aReleases.Files.First().Url)).ReturnsAsync(this.CreatePypiZip("a", "1.0", this.CreateMetadataString(new List() { b }))); + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(bReleases.Files.First().Url)).ReturnsAsync(this.CreatePypiZip("b", "1.0", this.CreateMetadataString(new List() { c }))); + + var dependencies = new List { specA }; + + var resolver = new SimplePythonResolver(this.simplePyPiClient.Object, this.loggerMock.Object); + + var resolveResult = await resolver.ResolveRootsAsync(this.recorderMock.Object, dependencies); + + Assert.IsNotNull(resolveResult); + + var expectedA = new PipGraphNode(new PipComponent("a", "1.0")); + var expectedB = new PipGraphNode(new PipComponent("b", "1.0")); + + expectedA.Children.Add(expectedB); + expectedB.Parents.Add(expectedA); + + Assert.IsTrue(this.CompareGraphs(resolveResult.First(), expectedA)); + this.simplePyPiClient.Verify(x => x.FetchPackageFileStreamAsync(It.IsAny()), Times.Exactly(2)); + } + + [TestMethod] + public async Task TestPipResolverBacktrackAsync() + { + var a = "a==1.0"; + var b = "b==1.0"; + var c = "c<=1.1"; + var cAlt = "c==1.0"; + + var specA = new PipDependencySpecification(a); + var specB = new PipDependencySpecification(b); + var specC = new PipDependencySpecification(c); + var specCAlt = new PipDependencySpecification(cAlt); + + var aReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.0", "bdist_wheel") }); + var bReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.0", "bdist_wheel") }); + var cReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.0", "bdist_wheel"), ("1.1", "bdist_wheel") }); + + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("a")))).ReturnsAsync(aReleases); + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("b")))).ReturnsAsync(bReleases); + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("c") && x.DependencySpecifiers.First().Equals("<=1.1")))).ReturnsAsync(cReleases); + + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(aReleases.Files.First().Url)).ReturnsAsync(this.CreatePypiZip("a", "1.0", this.CreateMetadataString(new List() { b, c }))); + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(bReleases.Files.First().Url)).ReturnsAsync(this.CreatePypiZip("b", "1.0", this.CreateMetadataString(new List() { cAlt }))); + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(cReleases.Files.First().Url)).ReturnsAsync(new MemoryStream()); + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(cReleases.Files.Last().Url)).ReturnsAsync(new MemoryStream()); + + var dependencies = new List { specA }; + + var resolver = new SimplePythonResolver(this.simplePyPiClient.Object, this.loggerMock.Object); + + var resolveResult = await resolver.ResolveRootsAsync(this.recorderMock.Object, dependencies); + + Assert.IsNotNull(resolveResult); + + var expectedA = new PipGraphNode(new PipComponent("a", "1.0")); + var expectedB = new PipGraphNode(new PipComponent("b", "1.0")); + var expectedC = new PipGraphNode(new PipComponent("c", "1.0")); + + expectedA.Children.Add(expectedB); + expectedA.Children.Add(expectedC); + expectedB.Parents.Add(expectedA); + expectedB.Children.Add(expectedC); + expectedC.Parents.Add(expectedA); + expectedC.Parents.Add(expectedB); + + Assert.IsTrue(this.CompareGraphs(resolveResult.First(), expectedA)); + this.simplePyPiClient.Verify(x => x.FetchPackageFileStreamAsync(It.IsAny()), Times.Exactly(4)); + } + + [TestMethod] + public async Task TestPipResolverVersionExtractionWithDifferentVersionFormatsAsync() + { + var a = "a==1.15.0"; + var b = "b==1.19"; + var c = "c==3.1.1"; + + var specA = new PipDependencySpecification(a); + var specB = new PipDependencySpecification(b); + var specC = new PipDependencySpecification(c); + + var aReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.15.0", "bdist_wheel") }); + var bReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.19", "bdist_wheel") }); + var cReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("3.1.1", "bdist_wheel") }); + + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("a")))).ReturnsAsync(aReleases); + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("b")))).ReturnsAsync(bReleases); + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("c")))).ReturnsAsync(cReleases); + + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(aReleases.Files.First().Url)).ReturnsAsync(this.CreatePypiZip("a", "1.15.0", this.CreateMetadataString(new List() { b }))); + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(bReleases.Files.First().Url)).ReturnsAsync(this.CreatePypiZip("b", "1.19", this.CreateMetadataString(new List() { c }))); + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(cReleases.Files.First().Url)).ReturnsAsync(new MemoryStream()); + + var dependencies = new List { specA }; + + var resolver = new SimplePythonResolver(this.simplePyPiClient.Object, this.loggerMock.Object); + + var resolveResult = await resolver.ResolveRootsAsync(this.recorderMock.Object, dependencies); + + Assert.IsNotNull(resolveResult); + + var expectedA = new PipGraphNode(new PipComponent("a", "1.15.0")); + var expectedB = new PipGraphNode(new PipComponent("b", "1.19")); + var expectedC = new PipGraphNode(new PipComponent("c", "3.1.1")); + + expectedA.Children.Add(expectedB); + expectedB.Parents.Add(expectedA); + expectedB.Children.Add(expectedC); + expectedC.Parents.Add(expectedB); + + Assert.IsTrue(this.CompareGraphs(resolveResult.First(), expectedA)); + } + + [TestMethod] + public async Task TestPipResolverVersionExtractionWithDifferentPackageTypesAsync() + { + var a = "a==1.20"; + var b = "b==1.0.0"; + var c = "c==1.0"; + + var specA = new PipDependencySpecification(a); + var specB = new PipDependencySpecification(b); + var specC = new PipDependencySpecification(c); + + var aReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.20", "bdist_egg") }); + var bReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.0.0", "sdist") }); + var cReleases = this.CreateSimplePypiProject(new List<(string, string)> { ("1.0", "bdist_wheel") }); + + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("a")))).ReturnsAsync(aReleases); + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("b")))).ReturnsAsync(bReleases); + this.simplePyPiClient.Setup(x => x.GetSimplePypiProjectAsync(It.Is(x => x.Name.Equals("c")))).ReturnsAsync(cReleases); + + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(aReleases.Files.First().Url)).ReturnsAsync(this.CreatePypiZip("a", "1.20", this.CreateMetadataString(new List() { b }))); + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(bReleases.Files.First().Url)).ReturnsAsync(this.CreatePypiZip("b", "1.0.0", this.CreateMetadataString(new List() { c }))); + this.simplePyPiClient.Setup(x => x.FetchPackageFileStreamAsync(cReleases.Files.First().Url)).ReturnsAsync(new MemoryStream()); + + var dependencies = new List { specA }; + + var resolver = new SimplePythonResolver(this.simplePyPiClient.Object, this.loggerMock.Object); + + var resolveResult = await resolver.ResolveRootsAsync(this.recorderMock.Object, dependencies); + + Assert.IsNotNull(resolveResult); + + var expectedA = new PipGraphNode(new PipComponent("a", "1.20")); + var expectedB = new PipGraphNode(new PipComponent("b", "1.0.0")); + + expectedA.Children.Add(expectedB); + expectedB.Parents.Add(expectedA); + + Assert.IsTrue(this.CompareGraphs(resolveResult.First(), expectedA)); + } + + private bool CompareGraphs(PipGraphNode a, PipGraphNode b) + { + var componentA = a.Value; + var componentB = b.Value; + + if (!string.Equals(componentA.Name, componentB.Name, StringComparison.OrdinalIgnoreCase) || + !string.Equals(componentA.Version, componentB.Version, StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + if (a.Children.Count != b.Children.Count) + { + return false; + } + + var valid = true; + + for (var i = 0; i < a.Children.Count; i++) + { + valid = this.CompareGraphs(a.Children[i], b.Children[i]); + } + + return valid; + } + + private SimplePypiProject CreateSimplePypiProject(IList<(string Version, string PackageTypes)> versionAndTypes) + { + var toReturn = new SimplePypiProject() { Files = new List() }; + + foreach ((var version, var packagetype) in versionAndTypes) + { + toReturn.Files.Add(this.CreateSimplePythonProjectRelease(version, packagetype)); + } + + return toReturn; + } + + private SimplePypiProjectRelease CreateSimplePythonProjectRelease(string version, string packageType = "bdist_wheel") + { + var releaseDict = new Dictionary(); + var fileExt = string.Empty; + if (packageType == "bdist_wheel") + { + fileExt = "-py2.py3-none-any.whl"; + } + else if (packageType == "sdist") + { + fileExt = ".tar.gz"; + } + else + { + fileExt = "-py3.5.egg"; + } + + return new SimplePypiProjectRelease { FileName = string.Format("google-cloud-secret-manager-{0}{1}", version, fileExt), Size = 1000, Url = new Uri($"https://{Guid.NewGuid()}") }; + } + + private string CreateMetadataString(IList dependency) + { + var metadataFile = @"Metadata-Version: 2.0 +Name: boto3 +Version: 1.10.9 +Summary: The AWS SDK for Python +Home-page: https://github.com/boto/boto3 +Author: Amazon Web Services +Author-email: UNKNOWN +License: Apache License 2.0 +Platform: UNKNOWN +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Natural Language :: English +Classifier: License :: OSI Approved :: Apache Software License +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.3 +Classifier: Programming Language :: Python :: 3.4 +Classifier: Programming Language :: Python :: 3.5 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7"; + + foreach (var dep in dependency) + { + metadataFile += Environment.NewLine + string.Format("Requires-Dist: {0}", dep); + } + + return metadataFile; + } + + private Stream CreatePypiZip(string name, string version, string content) + { + var stream = new MemoryStream(); + + using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, true)) + { + var entry = archive.CreateEntry($"{name.Replace('-', '_')}-{version}.dist-info/METADATA"); + + using var entryStream = entry.Open(); + + var templateBytes = Encoding.UTF8.GetBytes(content); + entryStream.Write(templateBytes); + } + + stream.Seek(0, SeekOrigin.Begin); + return stream; + } +}