diff --git a/providers/fetch/pypiFetch.js b/providers/fetch/pypiFetch.js index dbb56c48..01ab0842 100644 --- a/providers/fetch/pypiFetch.js +++ b/providers/fetch/pypiFetch.js @@ -26,7 +26,9 @@ class PyPiFetch extends AbstractFetch { request.url = spec.toUrl() super.handle(request) const file = this.createTempFile(request) - await this._getPackage(spec, registryData, file.name) + const downloaded = await this._getPackage(spec, registryData, file.name) + if (!downloaded) return request.markSkip('Missing ') + const dir = this.createTempDir(request) await this.decompress(file.name, dir.name) const hashes = await this.computeHashes(file.name) @@ -83,17 +85,16 @@ class PyPiFetch extends AbstractFetch { async _getPackage(spec, registryData, destination) { const releaseTypes = get(registryData, ['releases', spec.revision]) - const release = find(releaseTypes, entry => { - return entry.url && entry.url.length > 6 && entry.url.slice(-6) === 'tar.gz' - }) - if (!release) return + const release = find(releaseTypes, entry => entry.url?.endsWith('tar.gz') || entry.url?.endsWith('zip')) + if (!release) return false + return new Promise((resolve, reject) => { nodeRequest .get(release.url, (error, response) => { if (error) return reject(error) if (response.statusCode !== 200) reject(new Error(`${response.statusCode} ${response.statusMessage}`)) }) - .pipe(fs.createWriteStream(destination).on('finish', () => resolve(null))) + .pipe(fs.createWriteStream(destination).on('finish', () => resolve(true))) }) } } diff --git a/test/unit/providers/fetch/pypiFetchTests.js b/test/unit/providers/fetch/pypiFetchTests.js index 4b4b40fe..12982b9e 100644 --- a/test/unit/providers/fetch/pypiFetchTests.js +++ b/test/unit/providers/fetch/pypiFetchTests.js @@ -6,14 +6,16 @@ const sinon = require('sinon') const PypiFetch = require('../../../../providers/fetch/pypiFetch') const requestRetryWithDefaults = require('../../../../providers/fetch/requestRetryWithDefaults') const Request = require('../../../../ghcrawler/lib/request.js') -const pypiFetchOptions = { logger: { info: sinon.stub() }, cdFileLocation: 'test/fixtures/debian/fragment' } +const pypiFetchOptions = { logger: { info: sinon.stub() } } describe('pypiFetch handle function', () => { let sandbox = sinon.createSandbox() let requestGetStub + let fetch beforeEach(function () { requestGetStub = sandbox.stub(requestRetryWithDefaults, 'get') + fetch = PypiFetch(pypiFetchOptions) }) afterEach(function () { @@ -24,9 +26,21 @@ describe('pypiFetch handle function', () => { // Setup the stub to return an empty response (e.g. no body) requestGetStub.returns({}) - let fetch = PypiFetch(pypiFetchOptions) let result = await fetch.handle(new Request('pypi', 'cd:/pypi/pypi/-/reuse/0.8.1')) expect(result.outcome).to.be.equal('Missing ') }) + + it('returns missing when failed to find download url', async () => { + // release information in the registry data is empty + requestGetStub.returns({ + body: { + 'releases': { '1.10.0': [] } + }, + statusCode: 200 + }) + + let result = await fetch.handle(new Request('pypi', 'cd:/pypi/pypi/-/dnspython/1.10.0')) + expect(result.outcome).to.be.equal('Missing ') + }) })