From b1044c73025b61781742ec9b3362d67ea85297f8 Mon Sep 17 00:00:00 2001 From: sebasgomez238 <69322674+sebasgomez238@users.noreply.github.com> Date: Tue, 3 Oct 2023 13:20:36 -0400 Subject: [PATCH] Revert "Added spdx package component information from SPDX file (#766)" (#825) Co-authored-by: Sebastian Gomez --- .../TypedComponent/SpdxPackageComponent.cs | 39 ------ .../spdx/Contracts/CreationInfo.cs | 13 -- .../spdx/Contracts/SpdxExternalRefs.cs | 15 --- .../spdx/Contracts/SpdxFileData.cs | 34 ----- .../spdx/Contracts/SpdxPackage.cs | 34 ----- .../spdx/Spdx22ComponentDetector.cs | 64 ++++----- .../SPDX22ComponentDetectorTests.cs | 121 +----------------- 7 files changed, 32 insertions(+), 288 deletions(-) delete mode 100644 src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxPackageComponent.cs delete mode 100644 src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/CreationInfo.cs delete mode 100644 src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/SpdxExternalRefs.cs delete mode 100644 src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/SpdxFileData.cs delete mode 100644 src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/SpdxPackage.cs diff --git a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxPackageComponent.cs b/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxPackageComponent.cs deleted file mode 100644 index 09c09bbaa..000000000 --- a/src/Microsoft.ComponentDetection.Contracts/TypedComponent/SpdxPackageComponent.cs +++ /dev/null @@ -1,39 +0,0 @@ -namespace Microsoft.ComponentDetection.Contracts.TypedComponent; - -using PackageUrl; - -public class SpdxPackageComponent : TypedComponent -{ - public SpdxPackageComponent(string name, string version, string supplier, string copyrightText, string downloadLocation) - { - this.Name = this.ValidateRequiredInput(name, nameof(this.Name), nameof(ComponentType.Spdx)); - this.Version = this.ValidateRequiredInput(version, nameof(this.Version), nameof(ComponentType.Spdx)); - this.Supplier = this.ValidateRequiredInput(supplier, nameof(this.Supplier), nameof(ComponentType.Spdx)); - this.CopyrightText = this.ValidateRequiredInput(copyrightText, nameof(this.CopyrightText), nameof(ComponentType.Spdx)); - this.DownloadLocation = this.ValidateRequiredInput(downloadLocation, nameof(this.DownloadLocation), nameof(ComponentType.Spdx)); - } - - public SpdxPackageComponent(string name, string version, string supplier, string copyrightText, string downloadLocation, string packageUrl) - : this(name, version, supplier, copyrightText, downloadLocation) => this.PackageUrl = new PackageURL(packageUrl); - - private SpdxPackageComponent() - { - // reserved for deserialization - } - - public string CopyrightText { get; } - - public string DownloadLocation { get; } - - public override string Id => $"{this.Name} {this.Version} - {this.Type}"; - - public string Name { get; } - - public override PackageURL PackageUrl { get; } - - public string Supplier { get; } - - public override ComponentType Type => ComponentType.Spdx; - - public string Version { get; } -} diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/CreationInfo.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/CreationInfo.cs deleted file mode 100644 index 549ec6e81..000000000 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/CreationInfo.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace Microsoft.ComponentDetection.Detectors.Spdx.Contracts; - -using System.Collections.Generic; -using Newtonsoft.Json; - -public class CreationInfo -{ - [JsonProperty("created")] - public string Created { get; set; } - - [JsonProperty("creators")] - public IEnumerable Creators { get; set; } -} diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/SpdxExternalRefs.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/SpdxExternalRefs.cs deleted file mode 100644 index 26501c813..000000000 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/SpdxExternalRefs.cs +++ /dev/null @@ -1,15 +0,0 @@ -namespace Microsoft.ComponentDetection.Detectors.Spdx.Contracts; - -using Newtonsoft.Json; - -public class SpdxExternalRefs -{ - [JsonProperty("referenceCategory")] - public string Category { get; set; } - - [JsonProperty("referenceLocator")] - public string Locator { get; set; } - - [JsonProperty("referenceType")] - public string Type { get; set; } -} diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/SpdxFileData.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/SpdxFileData.cs deleted file mode 100644 index d2d4cd746..000000000 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/SpdxFileData.cs +++ /dev/null @@ -1,34 +0,0 @@ -namespace Microsoft.ComponentDetection.Detectors.Spdx.Contracts; - -using System.Collections.Generic; -using System.Linq; -using Newtonsoft.Json; - -public class SpdxFileData -{ - [JsonProperty("creationInfo")] - public CreationInfo CreationInfo { get; set; } - - [JsonProperty("dataLicense")] - public string DataLicense { get; set; } - - [JsonProperty("documentDescribes")] - public IEnumerable DocumentDescribes { get; set; } - - [JsonProperty("documentNamespace")] - public string DocumentNamespace { get; set; } - - [JsonProperty("name")] - public string Name { get; set; } - - [JsonProperty("packages")] - public IEnumerable Packages { get; set; } - - [JsonProperty(nameof(SPDXID))] - public string SPDXID { get; set; } - - [JsonProperty("spdxVersion")] - public string Version { get; set; } - - internal bool HasPackages() => this.Packages?.Count() > 0; -} diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/SpdxPackage.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/SpdxPackage.cs deleted file mode 100644 index 944df790d..000000000 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/Contracts/SpdxPackage.cs +++ /dev/null @@ -1,34 +0,0 @@ -namespace Microsoft.ComponentDetection.Detectors.Spdx.Contracts; - -using System.Collections.Generic; -using Newtonsoft.Json; - -public class SpdxPackage -{ - [JsonProperty("copyrightText")] - public string CopyrightText { get; set; } - - [JsonProperty("downloadLocation")] - public string DownloadLocation { get; set; } - - [JsonProperty("externalRefs")] - public IEnumerable ExternalRefs { get; set; } - - [JsonProperty("licenseConcluded")] - public string LicenseConcluded { get; set; } - - [JsonProperty("licenseDeclared")] - public string LicenseDeclared { get; set; } - - [JsonProperty("name")] - public string Name { get; set; } - - [JsonProperty(nameof(SPDXID))] - public string SPDXID { get; set; } - - [JsonProperty("supplier")] - public string Supplier { get; set; } - - [JsonProperty("versionInfo")] - public string Version { get; set; } -} diff --git a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs index 35fd5a303..fe710910d 100644 --- a/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs +++ b/src/Microsoft.ComponentDetection.Detectors/spdx/Spdx22ComponentDetector.cs @@ -1,4 +1,4 @@ -namespace Microsoft.ComponentDetection.Detectors.Spdx; +namespace Microsoft.ComponentDetection.Detectors.Spdx; using System; using System.Collections.Generic; @@ -9,9 +9,9 @@ namespace Microsoft.ComponentDetection.Detectors.Spdx; using Microsoft.ComponentDetection.Contracts; using Microsoft.ComponentDetection.Contracts.Internal; using Microsoft.ComponentDetection.Contracts.TypedComponent; -using Microsoft.ComponentDetection.Detectors.Spdx.Contracts; using Microsoft.Extensions.Logging; using Newtonsoft.Json; +using Newtonsoft.Json.Linq; /// /// Spdx22ComponentDetector discover SPDX SBOM files in JSON format and create components with the information about @@ -46,7 +46,6 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction { this.Logger.LogDebug("Discovered SPDX2.2 manifest file at: {ManifestLocation}", processRequest.ComponentStream.Location); var file = processRequest.ComponentStream; - var singleFileComponentRecorder = processRequest.SingleFileComponentRecorder; try { @@ -59,46 +58,30 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction using var reader = new JsonTextReader(sr); var serializer = new JsonSerializer(); - var spdxFileData = serializer.Deserialize(reader); - - if (spdxFileData == null) - { - this.Logger.LogWarning("Discovered SPDX file at {ManifestLocation} is not a valid document, skipping", processRequest.ComponentStream.Location); - return Task.CompletedTask; - } - - if (!this.IsSPDXVersionSupported(spdxFileData.Version)) - { - this.Logger.LogWarning("Discovered SPDX at {ManifestLocation} is not SPDX-2.2 document, skipping", processRequest.ComponentStream.Location); - return Task.CompletedTask; - } - - var sbomComponent = this.ConvertJObjectToSbomComponent(processRequest, spdxFileData, hash); - singleFileComponentRecorder.RegisterUsage(new DetectedComponent(sbomComponent)); - - if (spdxFileData.HasPackages()) + try { - foreach (var package in spdxFileData.Packages) + var document = serializer.Deserialize(reader); + if (document != null) { - SpdxPackageComponent spdxPackageComponent; - - var extRefLocator = package.ExternalRefs?.FirstOrDefault(x => x.Type == "purl")?.Locator; - if (extRefLocator is not null) + if (this.IsSPDXVersionSupported(document)) { - spdxPackageComponent = new SpdxPackageComponent(package.Name, package.Version, package.Supplier, package.CopyrightText, package.DownloadLocation, extRefLocator); + var sbomComponent = this.ConvertJObjectToSbomComponent(processRequest, document, hash); + processRequest.SingleFileComponentRecorder.RegisterUsage(new DetectedComponent(sbomComponent)); } else { - spdxPackageComponent = new SpdxPackageComponent(package.Name, package.Version, package.Supplier, package.CopyrightText, package.DownloadLocation); + this.Logger.LogWarning("Discovered SPDX at {ManifestLocation} is not SPDX-2.2 document, skipping", processRequest.ComponentStream.Location); } - - singleFileComponentRecorder.RegisterUsage(new DetectedComponent(spdxPackageComponent)); + } + else + { + this.Logger.LogWarning("Discovered SPDX file at {ManifestLocation} is not a valid document, skipping", processRequest.ComponentStream.Location); } } - } - catch (JsonException je) - { - this.Logger.LogWarning(je, "Unable to parse file at {ManifestLocation}, skipping", processRequest.ComponentStream.Location); + catch (JsonReaderException) + { + this.Logger.LogWarning("Unable to parse file at {ManifestLocation}, skipping", processRequest.ComponentStream.Location); + } } catch (Exception e) { @@ -108,13 +91,16 @@ protected override Task OnFileFoundAsync(ProcessRequest processRequest, IDiction return Task.CompletedTask; } - private bool IsSPDXVersionSupported(string version) => this.supportedSPDXVersions.Contains(version?.ToString(), StringComparer.OrdinalIgnoreCase); + private bool IsSPDXVersionSupported(JObject document) => this.supportedSPDXVersions.Contains(document["spdxVersion"]?.ToString(), StringComparer.OrdinalIgnoreCase); - private SpdxComponent ConvertJObjectToSbomComponent(ProcessRequest processRequest, SpdxFileData spdxFileData, string fileHash) + private SpdxComponent ConvertJObjectToSbomComponent(ProcessRequest processRequest, JObject document, string fileHash) { - var rootElements = spdxFileData.DocumentDescribes; + var sbomNamespace = document["documentNamespace"]?.ToString(); + var rootElements = document["documentDescribes"]?.ToObject(); + var name = document["name"]?.ToString(); + var spdxVersion = document["spdxVersion"]?.ToString(); - if (rootElements?.Count() > 1) + if (rootElements?.Length > 1) { this.Logger.LogWarning("SPDX file at {ManifestLocation} has more than one element in documentDescribes, first will be selected as root element.", processRequest.ComponentStream.Location); } @@ -126,7 +112,7 @@ private SpdxComponent ConvertJObjectToSbomComponent(ProcessRequest processReques var rootElementId = rootElements?.FirstOrDefault() ?? "SPDXRef-Document"; var path = processRequest.ComponentStream.Location; - var component = new SpdxComponent(spdxFileData.Version, new Uri(spdxFileData.DocumentNamespace), spdxFileData.Name, fileHash, rootElementId, path); + var component = new SpdxComponent(spdxVersion, new Uri(sbomNamespace), name, fileHash, rootElementId, path); return component; } diff --git a/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs b/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs index cef81055e..79336bdbb 100644 --- a/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs +++ b/test/Microsoft.ComponentDetection.Detectors.Tests/SPDX22ComponentDetectorTests.cs @@ -57,9 +57,9 @@ public async Task TestSbomDetector_SimpleSbomAsync() }], ""packages"": [ { - ""name"": ""package"", + ""name"": ""Test"", ""SPDXID"": ""SPDXRef-RootPackage"", - ""downloadLocation"": ""https://www.sbom.microsoft"", + ""downloadLocation"": ""NOASSERTION"", ""packageVerificationCode"": { ""packageVerificationCodeValue"": ""12fa1211046c12118936384b6c8683f1ac9b790a"" }, @@ -71,13 +71,6 @@ public async Task TestSbomDetector_SimpleSbomAsync() ""licenseDeclared"": ""NOASSERTION"", ""copyrightText"": ""NOASSERTION"", ""versionInfo"": ""1.0.0"", - ""externalRefs"": [ - { - ""referenceCategory"": ""PACKAGE-MANAGER"", - ""referenceType"": ""purl"", - ""referenceLocator"": ""pkg:nuget/openssl@2.10.0"" - } - ], ""supplier"": ""Organization: Microsoft"", ""hasFiles"": [""SPDXRef-File--.eslintrc.js-76586927C59544FB23BE1CF4D269882217EE21AB""] } @@ -115,124 +108,24 @@ public async Task TestSbomDetector_SimpleSbomAsync() var detectedComponents = componentRecorder.GetDetectedComponents(); var components = detectedComponents.ToList(); + var sbomComponent = (SpdxComponent)components.FirstOrDefault()?.Component; - if (!components.Any()) + if (sbomComponent is null) { - throw new AssertFailedException($"{nameof(SpdxComponent)} is null"); + throw new AssertFailedException($"{nameof(sbomComponent)} is null"); } - var sbomComponent = (SpdxComponent)components.First(x => x.Component.GetType() == typeof(SpdxComponent))?.Component; - var spdxPackageComponent = (SpdxPackageComponent)components.First(x => x.Component.GetType() == typeof(SpdxPackageComponent))?.Component; - - var spdxFilePath = Path.Combine(Path.GetTempPath(), spdxFileName); - #pragma warning disable CA5350 // Suppress Do Not Use Weak Cryptographic Algorithms because we use SHA1 intentionally in SPDX format var checksum = BitConverter.ToString(SHA1.HashData(Encoding.UTF8.GetBytes(spdxFile))).Replace("-", string.Empty).ToLower(); #pragma warning restore CA5350 - Assert.AreEqual(2, components.Count); + Assert.AreEqual(1, components.Count); Assert.AreEqual(sbomComponent.Name, "Test 1.0.0"); Assert.AreEqual(sbomComponent.RootElementId, "SPDXRef-RootPackage"); Assert.AreEqual(sbomComponent.DocumentNamespace, new Uri("https://sbom.microsoft/Test/1.0.0/61de1a5-57cc-4732-9af5-edb321b4a7ee")); Assert.AreEqual(sbomComponent.SpdxVersion, "SPDX-2.2"); Assert.AreEqual(sbomComponent.Checksum, checksum); - Assert.AreEqual(sbomComponent.Path, spdxFilePath); - - Assert.AreEqual(spdxPackageComponent.Name, "package"); - Assert.AreEqual(spdxPackageComponent.Version, "1.0.0"); - Assert.AreEqual(spdxPackageComponent.CopyrightText, "NOASSERTION"); - Assert.AreEqual(spdxPackageComponent.Supplier, "Organization: Microsoft"); - Assert.AreEqual(spdxPackageComponent.DownloadLocation, "https://www.sbom.microsoft"); - Assert.AreEqual(spdxPackageComponent.PackageUrl.ToString(), "pkg:nuget/openssl@2.10.0"); - Assert.AreEqual(spdxPackageComponent.Type, ComponentType.Spdx); - } - - [TestMethod] - public async Task TestSbomDetector_MissingExternalRefsAsync() - { - var spdxFile = /*lang=json,strict*/ @"{ - ""files"": [{ - ""fileName"": ""./.eslintrc.js"", - ""SPDXID"": ""SPDXRef-File--.eslintrc.js-76586927C59544FB23BE1CF4D269882217EE21AB"", - ""checksums"": [ - { - ""algorithm"": ""SHA256"", - ""checksumValue"": ""5c9c9d7eb9d31320bd621b3089ec0ae87d97e9ee7ed03cde2d20383928f72958"" - }, - { - ""algorithm"": ""SHA1"", - ""checksumValue"": ""76586927c59544fb23be1cf4d269882217ee21ab"" - } - ] - }], - ""packages"": [ - { - ""name"": ""package"", - ""SPDXID"": ""SPDXRef-RootPackage"", - ""downloadLocation"": ""https://www.sbom.microsoft"", - ""packageVerificationCode"": { - ""packageVerificationCodeValue"": ""12fa1211046c12118936384b6c8683f1ac9b790a"" - }, - ""filesAnalyzed"": true, - ""licenseConcluded"": ""NOASSERTION"", - ""licenseInfoFromFiles"": [ - ""NOASSERTION"" - ], - ""licenseDeclared"": ""NOASSERTION"", - ""copyrightText"": ""NOASSERTION"", - ""versionInfo"": ""1.0.0"", - ""supplier"": ""Organization: Microsoft"", - ""hasFiles"": [""SPDXRef-File--.eslintrc.js-76586927C59544FB23BE1CF4D269882217EE21AB""] - } - ], - ""relationships"": [ - { - ""relationshipType"": ""DESCRIBES"", - ""relatedSpdxElement"": ""SPDXRef-RootPackage"", - ""spdxElementId"": ""SPDXRef-DOCUMENT"" - } - ], - ""spdxVersion"": ""SPDX-2.2"", - ""dataLicense"": ""CC0-1.0"", - ""SPDXID"": ""SPDXRef-DOCUMENT"", - ""name"": ""Test 1.0.0"", - ""documentNamespace"": ""https://sbom.microsoft/Test/1.0.0/61de1a5-57cc-4732-9af5-edb321b4a7ee"", - ""creationInfo"": { - ""created"": ""2022-02-14T20:26:41Z"", - ""creators"": [ - ""Organization: Microsoft"", - ""Tool: Microsoft.SBOMTool-1.0.0"" - ] - }, - ""documentDescribes"": [ - ""SPDXRef-RootPackage"" - ] -}"; - - var spdxFileName = "manifest.spdx.json"; - var (scanResult, componentRecorder) = await this.DetectorTestUtility - .WithFile(spdxFileName, spdxFile) - .ExecuteDetectorAsync(); - - Assert.AreEqual(ProcessingResultCode.Success, scanResult.ResultCode); - - var detectedComponents = componentRecorder.GetDetectedComponents().ToList(); - - if (!detectedComponents.Any()) - { - throw new AssertFailedException($"{nameof(SpdxComponent)} is null"); - } - - var spdxPackageComponent = (SpdxPackageComponent)detectedComponents.First(x => x.Component.GetType() == typeof(SpdxPackageComponent))?.Component; - - Assert.IsNotNull(spdxPackageComponent); - Assert.AreEqual(spdxPackageComponent.Name, "package"); - Assert.AreEqual(spdxPackageComponent.Version, "1.0.0"); - Assert.AreEqual(spdxPackageComponent.CopyrightText, "NOASSERTION"); - Assert.AreEqual(spdxPackageComponent.Supplier, "Organization: Microsoft"); - Assert.AreEqual(spdxPackageComponent.DownloadLocation, "https://www.sbom.microsoft"); - Assert.IsNull(spdxPackageComponent.PackageUrl); - Assert.AreEqual(spdxPackageComponent.Type, ComponentType.Spdx); + Assert.AreEqual(sbomComponent.Path, Path.Combine(Path.GetTempPath(), spdxFileName)); } [TestMethod]