diff --git a/source/PackageCache.js b/source/PackageCache.js index 341cb27..1d38450 100644 --- a/source/PackageCache.js +++ b/source/PackageCache.js @@ -21,22 +21,36 @@ module.exports = { ) .split('\n')[0], - _synchronouslyPreparedPackages: new Set(), + _loadedESMExports: {}, + + _didRecentlyShowPreparingMessage: false, // Exposed - loadPackageBundle(rawPackageList, packageVersions) { - const packageList = this._normalizePackageList(rawPackageList, packageVersions); - - if (packageList.length > 0) { - this._prepareBundleForList(packageList); - } - }, - - loadPackageBundleSync(rawPackageList, packageVersions) { - const packageList = this._normalizePackageList(rawPackageList, packageVersions); + async preloadPackagesForScript(scriptParser) { + const packageList = this._normalizePackageList(scriptParser.requiredPackages, scriptParser.requiredPackageVersions); if (packageList.length > 0) { - this._prepareBundleForListSync(packageList); + // Prepare package bundle + const bundleIndexPath = this._bundlePathForList(packageList) + this.INDEX_FILE_NAME; + if (!fs.existsSync(bundleIndexPath)) { + this._showPreparingMessage(); + this._prepareBundleForListSync(packageList); + } + + // Preload all ESM exports + if (RuntimeVersion.isAtLeast('0.5')) { + for (let importPath of scriptParser.requiredPackages) { + const moduleExports = await this._loadESMModuleExports( + importPath, + scriptParser.requiredPackages, + scriptParser.requiredPackageVersions + ); + + if (moduleExports) { + this._loadedESMExports[importPath] = moduleExports; + } + } + } } }, @@ -50,74 +64,95 @@ module.exports = { return deletedBundleCount; }, - get(importPath, rawRequestedBundlePackageList, packageVersions) { + bundlePathForHash(packageHash) { + return this.PACKAGE_CACHE_PATH + packageHash + path.sep; + }, + + bundlePathForList(rawPackageList, packageVersions) { + const packageList = this._normalizePackageList(rawPackageList, packageVersions); + return this.bundlePathForHash(this._bundleHashForList(packageList)); + }, + + readableRequiredPackageListFor(rawPackageList, packageVersions) { + return this._normalizePackageList(rawPackageList, packageVersions).join(', '); + }, + + getModuleESMExports(importPath) { + return this._loadedESMExports[importPath] || null; + }, + + getModuleCommonJSExports(importPath, rawRequestedBundlePackageList, packageVersions) { + return this._loadCommonJSModuleExports(importPath, rawRequestedBundlePackageList, packageVersions); + }, + + // Internal + _prepareBundleForListSync(packageList) { + this._ensurePackageCacheFolderExists(); + + crossSpawn.sync('node', this._nodeArgumentsForList(packageList)); + }, + + /// Returns the exports of an ESM module. Doesn't do anything with the network. + async _loadESMModuleExports(importPath, rawRequestedBundlePackageList, packageVersions) { + const requestedBundlePackageList = this._normalizePackageList(rawRequestedBundlePackageList, packageVersions); + const packageName = this._packageNameForImportPath(importPath); + const flattenedImportPath = importPath.replace(/:/g, '/'); + + const bundleIndex = this._loadBundleIndex(requestedBundlePackageList, packageName); + + return await bundleIndex.importModuleAtPath(flattenedImportPath); + }, + + /// Returns the exports of a CommonJS module. Tries loading the requested bundle first, and fallbacks to a dedicated bundle for the package. + _loadCommonJSModuleExports(importPath, rawRequestedBundlePackageList, packageVersions) { const requestedBundlePackageList = this._normalizePackageList(rawRequestedBundlePackageList, packageVersions); const dedicatedBundlePackageList = this._dedicatedBundlePackageListFor(importPath, packageVersions); + const packageName = this._packageNameForImportPath(importPath); + const flattenedImportPath = importPath.replace(/:/g, '/'); let packageObject; // Try loading package from requested bundle if (requestedBundlePackageList) { - packageObject = this._getFromBundle(importPath, requestedBundlePackageList); + packageObject = + this._loadBundleIndex(requestedBundlePackageList, packageName) + .requireModuleAtPath(flattenedImportPath); } // Try loading package from dedicated bundle if (!packageObject) { - packageObject = this._getFromBundle(importPath, dedicatedBundlePackageList); + packageObject = + this._loadBundleIndex(dedicatedBundlePackageList, packageName) + .requireModuleAtPath(flattenedImportPath); } // If the dedicated bundle was present but incorrectly prepared, try again if (!packageObject) { - packageObject = this._getFromBundle(importPath, dedicatedBundlePackageList); + this._markBundleForDeletion(dedicatedBundlePackageList); + + packageObject = + this._loadBundleIndex(dedicatedBundlePackageList, packageName) + .requireModuleAtPath(flattenedImportPath); } // Couldn't load bundle if (!packageObject) { this._markBundleForDeletion(dedicatedBundlePackageList); - const packageName = this.packageNameForImportPath(importPath); - throw Error(`Package “${packageName}” could not be retrieved. Make sure its name is correct and that you are connected to the Internet.`); + const relativeImportPath = importPath.slice(packageName.length + 1); + if (relativeImportPath !== '') { + throw Error(`Module “${relativeImportPath}” from package “${packageName}” could not be loaded. Make sure the package name and import path are correct, and that you are connected to the Internet.`); + } else { + throw Error(`Package “${packageName}” could not be loaded. Make sure its name is correct and that you are connected to the Internet.`); + } } return packageObject; }, - bundlePathForHash(packageHash) { - return this.PACKAGE_CACHE_PATH + packageHash + path.sep; - }, - - bundlePathForList(rawPackageList, packageVersions) { - const packageList = this._normalizePackageList(rawPackageList, packageVersions); - return this.bundlePathForHash(this._bundleHashForList(packageList)); - }, - - readableRequiredPackageListFor(rawPackageList, packageVersions) { - return this._normalizePackageList(rawPackageList, packageVersions).join(', '); - }, - - // Internal - _prepareBundleForList(packageList) { - this._ensurePackageCacheFolderExists(); - - const preparationProcess = crossSpawn( - 'node', - this._nodeArgumentsForList(packageList), - { - detached: true, - stdio: 'ignore' - } - ); - preparationProcess.unref(); - }, - - _prepareBundleForListSync(packageList) { - this._ensurePackageCacheFolderExists(); - - crossSpawn.sync('node', this._nodeArgumentsForList(packageList)); - }, - - _getFromBundle(importPath, packageList) { - const packageName = this.packageNameForImportPath(importPath); + /// Tries to require the bundle's index. If initially unsuccessful, tries preparing the bundle once. + /// Preparing the bundle should only happen for dedicated bundles for CommonJS packages, as ESM packages can only be loaded ahead of time. + _loadBundleIndex(packageList, packageName) { const bundleIndexPath = this._bundlePathForList(packageList) + this.INDEX_FILE_NAME; let bundleIndex; @@ -128,10 +163,7 @@ module.exports = { bundleIndex = require(bundleIndexPath); } catch(e) { // Loading failed: try installing - if (!this._synchronouslyPreparedPackages.has(packageName)) { - this._synchronouslyPreparedPackages.add(packageName); - process.stdout.write('Preparing packages...\n'); - } + this._showPreparingMessage(); this._prepareBundleForListSync(packageList); } @@ -141,12 +173,11 @@ module.exports = { bundleIndex = require(bundleIndexPath); } catch(e) { // Something is very wrong: preparePackageCacheBundle.js shouldn't ever fail, as it installs all packages as optional dependencies - throw Error(`Package “${packageName}” could not be retrieved: the package cache bundle preparation process is failing. Make sure the names of your packages are correct.`); + throw Error(`Package “${packageName}” could not be retrieved: the package cache bundle preparation process is failing. Make sure the names of your packages are correct and that you are connected to the Internet.`); } } - const flattenedImportPath = importPath.replace(/:/g, '/'); - return bundleIndex(flattenedImportPath); + return bundleIndex; }, _markBundleForDeletion(packageList) { @@ -166,7 +197,7 @@ module.exports = { }, // // Tools - packageNameForImportPath(importPath) { + _packageNameForImportPath(importPath) { if (RuntimeVersion.isLowerThan('0.3')) { importPath = importPath.replace(/\//g, ':'); } @@ -177,7 +208,7 @@ module.exports = { _normalizePackageList(packageList, packageVersions = {}) { const normalized = packageList .map(importPath => { - const packageName = this.packageNameForImportPath(importPath); + const packageName = this._packageNameForImportPath(importPath); const packageVersion = packageVersions[packageName]; if (packageVersion) { @@ -214,5 +245,16 @@ module.exports = { path.join(__dirname, 'preparePackageCacheBundle.js'), ...packageList ]; + }, + + _showPreparingMessage() { + if (!this._didRecentlyShowPreparingMessage) { + process.stdout.write('Preparing packages...\n'); + + this._didRecentlyShowPreparingMessage = true; + setImmediate(() => { + this._didRecentlyShowPreparingMessage = false; + }); + } } }; diff --git a/source/ScriptParser.js b/source/ScriptParser.js index 88ed54e..b6363af 100644 --- a/source/ScriptParser.js +++ b/source/ScriptParser.js @@ -105,9 +105,9 @@ module.exports = class ScriptParser { return false; } - pinPackageVersions() { + async pinPackageVersions() { // Run a synchronous install for the detected packages - PackageCache.loadPackageBundleSync(this.requiredPackages, this.requiredPackageVersions); + await PackageCache.preloadPackagesForScript(this); // Load package lock file; parse it for the versions const bundlePath = PackageCache.bundlePathForList(this.requiredPackages, this.requiredPackageVersions); diff --git a/source/ScriptRunner.js b/source/ScriptRunner.js index 6733d3a..16c6a61 100644 --- a/source/ScriptRunner.js +++ b/source/ScriptRunner.js @@ -13,7 +13,7 @@ const PackageCache = require('./PackageCache'); const ScriptEnvironment = require('./ScriptEnvironment'); module.exports = { - run(scriptSource, scriptPath, args) { + async run(scriptSource, scriptPath, args) { const parser = new ScriptParser(scriptSource); const requiredPackages = parser.requiredPackages; @@ -34,8 +34,8 @@ module.exports = { preparedScriptPath = path.join(stagePath, path.basename(scriptPath)); fs.writeFileSync(preparedScriptPath, parser.preparedSource); - // Preload packages asynchronously - PackageCache.loadPackageBundle(requiredPackages, requiredPackageVersions); + // Preload packages + await PackageCache.preloadPackagesForScript(parser); // Execute script require(preparedScriptPath); diff --git a/source/exposed-modules/injected/npm.js b/source/exposed-modules/injected/npm.js index 433af18..136eb33 100644 --- a/source/exposed-modules/injected/npm.js +++ b/source/exposed-modules/injected/npm.js @@ -4,11 +4,55 @@ const ScriptEnvironment = require('../../ScriptEnvironment'); const PackageCache = require('../../PackageCache'); module.exports = new Proxy({}, { - get(self, importPath) { - return PackageCache.get( - importPath, - ScriptEnvironment.defaultBundlePackageList, - ScriptEnvironment.requiredPackageVersions - ); + get(target, property) { + return proxyForModule(property); } }); + +function proxyForModule(importPath) { + const esmExports = PackageCache.getModuleESMExports(importPath); + let commonJSExports; + + function getCommonJSExports() { + // Load CommonJS exports on demand, since the load can have side-effects + if (!commonJSExports) { + commonJSExports = PackageCache.getModuleCommonJSExports( + importPath, + ScriptEnvironment.defaultBundlePackageList, + ScriptEnvironment.requiredPackageVersions + ); + } + + return commonJSExports; + } + + const proxyTarget = esmExports + ? esmExports.default || {} + : getCommonJSExports(); + + return new Proxy(proxyTarget, { + get(target, property) { + if (property === 'unmodifiedDefaultExport') { + // Return unmodified default ESM export or CommonJS exports + return target; + } else if (esmExports && property in esmExports) { + // Return named ESM export + return esmExports[property]; + } else if (esmExports && 'default' in esmExports) { + // Return property of default ESM export + return bindProperty(esmExports.default, property); + } else { + // Return property of CommonJS exports + return bindProperty(getCommonJSExports(), property); + } + } + }); +} + +function bindProperty(object, property) { + if (typeof property === 'function') { + return object[property].bind(object); + } else { + return object[property]; + } +} diff --git a/source/packageCacheBundleIndexFile.template.js b/source/packageCacheBundleIndexFile.template.js index 2072e1b..2b15416 100644 --- a/source/packageCacheBundleIndexFile.template.js +++ b/source/packageCacheBundleIndexFile.template.js @@ -1,9 +1,18 @@ -// v0 +// v1 -module.exports = function getPackage(path) { - try { - return require(path); - } catch(e) { - return null; +module.exports = { + async importModuleAtPath(path) { + try { + return await import(path); + } catch(e) { + return null; + } + }, + requireModuleAtPath(path) { + try { + return require(path); + } catch(e) { + return null; + } } }; diff --git a/source/tasklemon.js b/source/tasklemon.js index a1c0767..15c93e5 100755 --- a/source/tasklemon.js +++ b/source/tasklemon.js @@ -140,8 +140,8 @@ function exitIfContainsInvalidArguments(args) { if (actionsToPerform.pinPackageVersions) { // Pin const parser = new ScriptParser(scriptFile.source); - const pinnedInfo = parser.pinPackageVersions(); - scriptFile.source = parser.source; + const pinnedInfo = await parser.pinPackageVersions(); + scriptFile.source = parser.source; // eslint-disable-line require-atomic-updates // Display outcome if (pinnedInfo.length > 0) { @@ -159,7 +159,7 @@ function exitIfContainsInvalidArguments(args) { if (actionsToPerform.preloadPackages) { const parser = new ScriptParser(scriptFile.source); if (parser.requiredPackages.length > 0) { - PackageCache.loadPackageBundleSync(parser.requiredPackages, parser.requiredPackageVersions); + await PackageCache.preloadPackagesForScript(parser); const readablePackageList = PackageCache.readableRequiredPackageListFor(parser.requiredPackages, parser.requiredPackageVersions); process.stdout.write(`Preloaded ${readablePackageList}.\n`); diff --git a/spec/general/ScriptParserSpec.js b/spec/general/ScriptParserSpec.js index a729c75..7a90549 100644 --- a/spec/general/ScriptParserSpec.js +++ b/spec/general/ScriptParserSpec.js @@ -116,7 +116,7 @@ cli.tell(await npm['username']()); `; const scriptParser = new ScriptParser(scriptSource); - scriptParser.pinPackageVersions(); + await scriptParser.pinPackageVersions(); expect(scriptParser.source).toMatch( new RegExp( @@ -133,7 +133,7 @@ cli.tell(await npm['username']()); `; const scriptParser = new ScriptParser(scriptSource); - scriptParser.pinPackageVersions(); + await scriptParser.pinPackageVersions(); expect(scriptParser.source).toMatch( new RegExp( diff --git a/spec/npm/npmSpec.js b/spec/npm/npmSpec.js index 0fcd48d..552bbc1 100644 --- a/spec/npm/npmSpec.js +++ b/spec/npm/npmSpec.js @@ -4,10 +4,10 @@ describe('npm', function() { beforeEach(function() { testEnv = this.getTestEnv(); }); - - it('should install and run simply named packages', async function() { + + it('should expose the default export of named ESM packages', async function() { const scriptSource = ` - const uniqueCount = npm.dedupe([1, 2, 2, 3]).length; + const uniqueCount = npm['array-union']([1, 2], [2, 3]).length; process.stdout.write(String(uniqueCount)); `; @@ -15,9 +15,35 @@ describe('npm', function() { expect(scriptOutput).toBe('3'); }); - - it('should install and run scoped packages', async function() { + + it('should expose the named exports of named ESM packages', async function() { const scriptSource = ` + const yes = npm['ts-extras'].isDefined(true); + const no = npm['ts-extras'].isDefined(undefined); + + if (yes && !no) { + process.stdout.write('success'); + } + `; + + let scriptOutput = await testEnv.runLemonScript(scriptSource); + + expect(scriptOutput).toBe('success'); + }); + + it('should expose the exports of CommonJS packages', async function() { + const scriptSource = ` + const uniqueCount = npm.dedupe([1, 2, 2, 3]).length; + process.stdout.write(String(uniqueCount)); + `; + + let scriptOutput = await testEnv.runLemonScript(scriptSource); + + expect(scriptOutput).toBe('3'); + }); + + it('should expose scoped packages', async function() { + const scriptSource = `// tl:require: @stryker-mutator/core@4.6.0 const stryker = new npm['@stryker-mutator/core'].Stryker({ concurrency: 4 }); process.stdout.write(String(stryker.cliOptions.concurrency)); `; @@ -26,7 +52,7 @@ describe('npm', function() { expect(scriptOutput).toBe('4'); }); - + it('should recognize pinned versions for scoped packages', async function() { // The require header must be in column 0 of the script, thus the unusual formatting const scriptSource = `// tl:require: @octokit/core@3.2.4 @@ -37,23 +63,34 @@ describe('npm', function() { expect(scriptOutput).toBe('3.2.4'); }); + + it('should expose nested ESM exports', async function() { + const scriptSource = ` + const Api = npm['telegram:tl:index.js'].Api; + + if ('Message' in Api) { + process.stdout.write('success'); + } + `; + + let scriptOutput = await testEnv.runLemonScript(scriptSource); + + expect(scriptOutput).toBe('success'); + }); - it('should allow require-access to sub files', async function() { + it('should expose nested CommonJS exports', async function() { const scriptSource = `// tl:require: uuid@3.3.0 const uuid_v1 = npm['uuid:v1']; const uuid_v4 = npm['uuid:v4']; - process.stdout.write(JSON.stringify({ - v1: uuid_v1.toString(), - v4: uuid_v4.toString() - })); + if (uuid_v1().length === 36 && uuid_v4().length === 36) { + process.stdout.write('success'); + } `; let scriptOutput = await testEnv.runLemonScript(scriptSource); - let parsedScriptOutput = JSON.parse(scriptOutput); - expect(parsedScriptOutput.v1).toContain('function v1'); - expect(parsedScriptOutput.v4).toContain('function v4'); + expect(scriptOutput).toBe('success'); }); describe('(pre-0.3)', function() { @@ -62,18 +99,15 @@ describe('npm', function() { #require uuid@3.3.0 const uuid_v1 = npm['uuid/v1']; const uuid_v4 = npm['uuid/v4']; - - process.stdout.write(JSON.stringify({ - v1: uuid_v1.toString(), - v4: uuid_v4.toString() - })); + + if (uuid_v1().length === 36 && uuid_v4().length === 36) { + process.stdout.write('success'); + } `; let scriptOutput = await testEnv.runLemonScript(scriptSource, [], ['--no-pin']); - let parsedScriptOutput = JSON.parse(scriptOutput); - expect(parsedScriptOutput.v1).toContain('function v1'); - expect(parsedScriptOutput.v4).toContain('function v4'); + expect(scriptOutput).toBe('success'); }); }); });