-
Notifications
You must be signed in to change notification settings - Fork 89
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Users/oakeredolu/newpipdetector (#684)
* Added a new defaultOff component detector for the new pypi api.
- Loading branch information
Showing
6 changed files
with
604 additions
and
11 deletions.
There are no files selected for viewing
133 changes: 133 additions & 0 deletions
133
src/Microsoft.ComponentDetection.Detectors/pip/SimplePipComponentDetector.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,133 @@ | ||
| namespace Microsoft.ComponentDetection.Detectors.Pip; | ||
|
|
||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.Linq; | ||
| using System.Reactive.Linq; | ||
| using System.Threading.Tasks; | ||
| using Microsoft.ComponentDetection.Contracts; | ||
| using Microsoft.ComponentDetection.Contracts.Internal; | ||
| using Microsoft.ComponentDetection.Contracts.TypedComponent; | ||
| using Microsoft.Extensions.Logging; | ||
|
|
||
| public class SimplePipComponentDetector : FileComponentDetector, IDefaultOffComponentDetector | ||
| { | ||
| private readonly IPythonCommandService pythonCommandService; | ||
| private readonly ISimplePythonResolver pythonResolver; | ||
|
|
||
| public SimplePipComponentDetector( | ||
| IComponentStreamEnumerableFactory componentStreamEnumerableFactory, | ||
| IObservableDirectoryWalkerFactory walkerFactory, | ||
| IPythonCommandService pythonCommandService, | ||
| ISimplePythonResolver pythonResolver, | ||
| ILogger<SimplePipComponentDetector> logger) | ||
| { | ||
| this.ComponentStreamEnumerableFactory = componentStreamEnumerableFactory; | ||
| this.Scanner = walkerFactory; | ||
| this.pythonCommandService = pythonCommandService; | ||
| this.pythonResolver = pythonResolver; | ||
| this.Logger = logger; | ||
| } | ||
|
|
||
| public override string Id => "SimplePip"; | ||
|
|
||
| public override IList<string> SearchPatterns => new List<string> { "setup.py", "requirements.txt" }; | ||
|
|
||
| public override IEnumerable<string> Categories => new List<string> { "Python" }; | ||
|
|
||
| public override IEnumerable<ComponentType> SupportedComponentTypes { get; } = new[] { ComponentType.Pip }; | ||
|
|
||
| public override int Version { get; } = 1; | ||
|
|
||
| protected override async Task<IObservable<ProcessRequest>> OnPrepareDetectionAsync(IObservable<ProcessRequest> processRequests, IDictionary<string, string> detectorArgs) | ||
| { | ||
| this.CurrentScanRequest.DetectorArgs.TryGetValue("Pip.PythonExePath", out var pythonExePath); | ||
| if (!await this.pythonCommandService.PythonExistsAsync(pythonExePath)) | ||
| { | ||
| this.Logger.LogInformation($"No python found on system. Python detection will not run."); | ||
|
|
||
| return Enumerable.Empty<ProcessRequest>().ToObservable(); | ||
| } | ||
|
|
||
| return processRequests; | ||
| } | ||
|
|
||
| protected override async Task OnFileFoundAsync(ProcessRequest processRequest, IDictionary<string, string> detectorArgs) | ||
| { | ||
| this.CurrentScanRequest.DetectorArgs.TryGetValue("Pip.PythonExePath", out var pythonExePath); | ||
| var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; | ||
| var file = processRequest.ComponentStream; | ||
|
|
||
| try | ||
| { | ||
| var initialPackages = await this.pythonCommandService.ParseFileAsync(file.Location, pythonExePath); | ||
| var listedPackage = initialPackages.Where(tuple => tuple.PackageString != null) | ||
| .Select(tuple => tuple.PackageString) | ||
| .Where(x => !string.IsNullOrWhiteSpace(x)) | ||
| .Select(x => new PipDependencySpecification(x)) | ||
| .Where(x => !x.PackageIsUnsafe()) | ||
| .ToList(); | ||
|
|
||
| var roots = await this.pythonResolver.ResolveRootsAsync(singleFileComponentRecorder, listedPackage); | ||
|
|
||
| RecordComponents( | ||
| singleFileComponentRecorder, | ||
| roots); | ||
|
|
||
| initialPackages.Where(tuple => tuple.Component != null) | ||
| .Select(tuple => new DetectedComponent(tuple.Component)) | ||
| .ToList() | ||
| .ForEach(gitComponent => singleFileComponentRecorder.RegisterUsage(gitComponent, isExplicitReferencedDependency: true)); | ||
| } | ||
| catch (Exception e) | ||
| { | ||
| this.Logger.LogError(e, "Error while parsing pip components in {File}", file.Location); | ||
| } | ||
| } | ||
|
|
||
| private static void RecordComponents( | ||
| ISingleFileComponentRecorder recorder, | ||
| IList<PipGraphNode> roots) | ||
| { | ||
| var nonRoots = new Queue<(DetectedComponent, PipGraphNode)>(); | ||
|
|
||
| var explicitRoots = roots.Select(a => a.Value).ToHashSet(); | ||
|
|
||
| foreach (var root in roots) | ||
| { | ||
| var rootDetectedComponent = new DetectedComponent(root.Value); | ||
|
|
||
| recorder.RegisterUsage( | ||
| rootDetectedComponent, | ||
| isExplicitReferencedDependency: true); | ||
|
|
||
| foreach (var child in root.Children) | ||
| { | ||
| nonRoots.Enqueue((rootDetectedComponent, child)); | ||
| } | ||
| } | ||
|
|
||
| var registeredIds = new HashSet<string>(); | ||
|
|
||
| while (nonRoots.Count > 0) | ||
| { | ||
| var (parent, item) = nonRoots.Dequeue(); | ||
|
|
||
| var detectedComponent = new DetectedComponent(item.Value); | ||
|
|
||
| recorder.RegisterUsage( | ||
| detectedComponent, | ||
| parentComponentId: parent.Component.Id); | ||
|
|
||
| if (!registeredIds.Contains(detectedComponent.Component.Id)) | ||
| { | ||
| foreach (var child in item.Children) | ||
| { | ||
| nonRoots.Enqueue((detectedComponent, child)); | ||
| } | ||
|
|
||
| registeredIds.Add(detectedComponent.Component.Id); | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
22 changes: 22 additions & 0 deletions
22
src/Microsoft.ComponentDetection.Orchestrator/Experiments/Configs/SimplePipExperiment.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| namespace Microsoft.ComponentDetection.Orchestrator.Experiments.Configs; | ||
|
|
||
| using Microsoft.ComponentDetection.Contracts; | ||
| using Microsoft.ComponentDetection.Detectors.Pip; | ||
|
|
||
| /// <summary> | ||
| /// Validating the <see cref="SimplePipComponentDetector"/>. | ||
| /// </summary> | ||
| public class SimplePipExperiment : IExperimentConfiguration | ||
| { | ||
| /// <inheritdoc /> | ||
| public string Name => "NewPipDetector"; | ||
|
|
||
| /// <inheritdoc /> | ||
| public bool IsInControlGroup(IComponentDetector componentDetector) => componentDetector is PipComponentDetector; | ||
|
|
||
| /// <inheritdoc /> | ||
| public bool IsInExperimentGroup(IComponentDetector componentDetector) => componentDetector is SimplePipComponentDetector; | ||
|
|
||
| /// <inheritdoc /> | ||
| public bool ShouldRecord(IComponentDetector componentDetector, int numComponents) => true; | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.