From 44e999810e8a40cdb007c623c38acbc54e0cf987 Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Diederich Date: Wed, 8 Jun 2022 17:24:31 +0200 Subject: [PATCH 01/18] Updated depedencies without braking changes for vulnerabilities Used "npm audit fix". --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 646e97241..6927bd7ee 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,8 @@ classes/ node_modules/ yarn-error.log src/docs/.vuepress/dist/ +bin/ .DS_Store jd-gui.cfg -bin/ .vscode/ +package-lock.json From 8096419dd2f37bacdcdf3f754f94c8966861e436 Mon Sep 17 00:00:00 2001 From: Jonathan Chapman and Jim Cifarelli Date: Mon, 16 Mar 2015 11:23:43 -0400 Subject: [PATCH 02/18] Displaying `warn` level messages when adding a file attempts to overwrite an existing resource --- .../shadow/tasks/ShadowCopyAction.groovy | 45 ++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy index 128dc3cce..e37f00cf6 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy @@ -199,7 +199,7 @@ class ShadowCopyAction implements CopyAction { private final Set unused private final ShadowStats stats - private Set visitedFiles = new HashSet() + private Map visitedFiles = new HashMap() StreamAction(ZipOutputStream zipOutStr, String encoding, List transformers, List relocators, PatternSet patternSet, Set unused, @@ -216,8 +216,25 @@ class ShadowCopyAction implements CopyAction { } } - private boolean recordVisit(RelativePath path) { - return visitedFiles.add(path.pathString) + private boolean recordVisit(path, size, originJar) { + if (visitedFiles.containsKey(path.toString())) { + return false + } + + if (originJar == null) { + originJar = "" + } + + visitedFiles.put(path.toString(), [size: size, originJar: originJar]) + return true + } + + private boolean recordVisit(path) { + return recordVisit(path.toString(), 0, null) + } + + private boolean recordVisit(FileCopyDetails fileCopyDetails) { + return recordVisit(fileCopyDetails.relativePath, fileCopyDetails.size, null) } @Override @@ -240,7 +257,7 @@ class ShadowCopyAction implements CopyAction { } else if (isClass && !isUnused(fileDetails.path)) { remapClass(fileDetails) } - recordVisit(fileDetails.relativePath) + recordVisit(fileDetails) } catch (Exception e) { throw new GradleException(String.format("Could not add %s to ZIP '%s'.", fileDetails, zipFile), e) } @@ -254,15 +271,15 @@ class ShadowCopyAction implements CopyAction { ZipFile archive = new ZipFile(fileDetails.file) try { List archiveElements = archive.entries.collect { - new ArchiveFileTreeElement(new RelativeArchivePath(it)) + new ArchiveFileTreeElement(new RelativeArchivePath(it, fileDetails)) } Spec patternSpec = patternSet.getAsSpec() List filteredArchiveElements = archiveElements.findAll { ArchiveFileTreeElement archiveElement -> - patternSpec.isSatisfiedBy(archiveElement.asFileTreeElement()) + patternSpec.isSatisfiedBy(archiveElement) } filteredArchiveElements.each { ArchiveFileTreeElement archiveElement -> if (archiveElement.relativePath.file) { - visitArchiveFile(archiveElement, archive) + visitArchiveFile(archiveElement, archive, fileDetails) } } } finally { @@ -272,21 +289,29 @@ class ShadowCopyAction implements CopyAction { } private void visitArchiveDirectory(RelativeArchivePath archiveDir) { - if (recordVisit(archiveDir)) { + if (recordVisit(archiveDir.toString())) { zipOutStr.putNextEntry(archiveDir.entry) zipOutStr.closeEntry() } } - private void visitArchiveFile(ArchiveFileTreeElement archiveFile, ZipFile archive) { + private void visitArchiveFile(ArchiveFileTreeElement archiveFile, ZipFile archive, FileCopyDetails fileDetails) { def archiveFilePath = archiveFile.relativePath + def archiveFileSize = archiveFile.size + if (archiveFile.classFile || !isTransformable(archiveFile)) { - if (recordVisit(archiveFilePath) && !isUnused(archiveFilePath.entry.name)) { + if (recordVisit(archiveFilePath.toString(), archiveFileSize, fileDetails.relativePath) && !isUnused(archiveFilePath.entry.name)) { if (!remapper.hasRelocators() || !archiveFile.classFile) { copyArchiveEntry(archiveFilePath, archive) } else { remapClass(archiveFilePath, archive) } + } else { + def archiveFileInVisitedFiles = visitedFiles.get(archiveFilePath.toString()) + if (archiveFileInVisitedFiles && (archiveFileInVisitedFiles.size != fileDetails.size) && !archiveFilePath.toString().startsWith('META-INF/')) { + log.warn("IGNORING ${archiveFilePath} from ${fileDetails.relativePath}, size is different (${fileDetails.size} vs ${archiveFileInVisitedFiles.size})") + log.warn(" --> origin JAR was ${archiveFileInVisitedFiles.originJar}") + } } } else { transform(archiveFile, archive) From 63282c35539bc04f96c4384f00c38be22d3cdccd Mon Sep 17 00:00:00 2001 From: Jonathan Chapman and Jim Cifarelli Date: Thu, 19 Mar 2015 09:56:29 -0400 Subject: [PATCH 03/18] Adding tests, not filtering META-INF/ by default --- .../shadow/tasks/ShadowCopyAction.groovy | 8 +++++-- .../plugins/shadow/ShadowPluginSpec.groovy | 24 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy index e37f00cf6..c41caadb7 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy @@ -308,9 +308,13 @@ class ShadowCopyAction implements CopyAction { } } else { def archiveFileInVisitedFiles = visitedFiles.get(archiveFilePath.toString()) - if (archiveFileInVisitedFiles && (archiveFileInVisitedFiles.size != fileDetails.size) && !archiveFilePath.toString().startsWith('META-INF/')) { + if (archiveFileInVisitedFiles && (archiveFileInVisitedFiles.size != fileDetails.size)) { log.warn("IGNORING ${archiveFilePath} from ${fileDetails.relativePath}, size is different (${fileDetails.size} vs ${archiveFileInVisitedFiles.size})") - log.warn(" --> origin JAR was ${archiveFileInVisitedFiles.originJar}") + if (archiveFileInVisitedFiles.originJar) { + log.warn(" --> origin JAR was ${archiveFileInVisitedFiles.originJar}") + } else { + log.warn(" --> file originated from project sourcecode") + } } } } else { diff --git a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy index a19284e5a..9f74f22d8 100644 --- a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy +++ b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy @@ -132,6 +132,30 @@ class ShadowPluginSpec extends PluginSpecification { assert output.exists() } + def 'warns when a file is masked by a previously shadowed resource'() { + given: + URL artifact = this.class.classLoader.getResource('test-artifact-1.0-SNAPSHOT.jar') + URL project = this.class.classLoader.getResource('test-project-1.0-SNAPSHOT.jar') + + buildFile << """ + |task shadow(type: ${ShadowJar.name}) { + | destinationDir = buildDir + | baseName = 'shadow' + | from('${artifact.path}') + | from('${project.path}') + |} + """.stripMargin() + + when: + runner.arguments << 'shadow' + ExecutionResult result = runner.run() + + then: + success(result) + assert result.standardOutput =~ /IGNORING META-INF\/MANIFEST\.MF from test-artifact-1\.0-SNAPSHOT\.jar, size is different \(3115 vs 25\)\s --> file originated from project sourcecode/ + assert result.standardOutput =~ /IGNORING META-INF\/MANIFEST\.MF from test-project-1\.0-SNAPSHOT\.jar, size is different \(3906 vs 25\)\s --> file originated from project sourcecode/ + } + def 'include project sources'() { given: file('src/main/java/shadow/Passed.java') << ''' From d26148f57ccce04f63cbe3ec5e7a56062caf7eb3 Mon Sep 17 00:00:00 2001 From: Another User Date: Fri, 10 Jun 2022 12:37:09 +0200 Subject: [PATCH 04/18] Cleaned up "Collision logging" and added StandardFilesMergeTransformer to merge standard files --- .../shadow/tasks/ShadowCopyAction.groovy | 61 ++++++--- .../plugins/shadow/tasks/ShadowJar.java | 123 ++++++++++++----- .../StandardFilesMergeTransformer.groovy | 127 ++++++++++++++++++ .../transformers/TransformerContext.groovy | 5 + .../StandardFilesMergeTransformerTest.groovy | 61 +++++++++ 5 files changed, 326 insertions(+), 51 deletions(-) create mode 100644 src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformer.groovy create mode 100644 src/test/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformerTest.groovy diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy index c41caadb7..5a2585a1c 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy @@ -5,6 +5,7 @@ import com.github.jengelman.gradle.plugins.shadow.impl.RelocatorRemapper import com.github.jengelman.gradle.plugins.shadow.internal.UnusedTracker import com.github.jengelman.gradle.plugins.shadow.internal.ZipCompressor import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator +import com.github.jengelman.gradle.plugins.shadow.transformers.StandardFilesMergeTransformer import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext import groovy.util.logging.Slf4j @@ -19,12 +20,10 @@ import org.gradle.api.Action import org.gradle.api.GradleException import org.gradle.api.UncheckedIOException import org.gradle.api.file.FileCopyDetails -import org.gradle.api.file.FilePermissions import org.gradle.api.file.FileTreeElement import org.gradle.api.file.RelativePath import org.gradle.api.internal.DocumentationRegistry import org.gradle.api.internal.file.CopyActionProcessingStreamAction -import org.gradle.api.internal.file.DefaultFilePermissions import org.gradle.api.internal.file.DefaultFileTreeElement import org.gradle.api.internal.file.copy.CopyAction import org.gradle.api.internal.file.copy.CopyActionProcessingStream @@ -40,6 +39,7 @@ import org.objectweb.asm.ClassVisitor import org.objectweb.asm.ClassWriter import org.objectweb.asm.commons.ClassRemapper +import javax.annotation.Nullable import java.util.zip.ZipException @Slf4j @@ -59,9 +59,9 @@ class ShadowCopyAction implements CopyAction { private final UnusedTracker unusedTracker ShadowCopyAction(File zipFile, ZipCompressor compressor, DocumentationRegistry documentationRegistry, - String encoding, List transformers, List relocators, - PatternSet patternSet, ShadowStats stats, - boolean preserveFileTimestamps, boolean minimizeJar, UnusedTracker unusedTracker) { + String encoding, List transformers, List relocators, + PatternSet patternSet, ShadowStats stats, + boolean preserveFileTimestamps, boolean minimizeJar, UnusedTracker unusedTracker) { this.zipFile = zipFile this.compressor = compressor @@ -150,7 +150,7 @@ class ShadowCopyAction implements CopyAction { private static void withResource(T resource, Action action) { try { action.execute(resource) - } catch(Throwable t) { + } catch (Throwable t) { try { resource.close() } catch (IOException e) { @@ -199,11 +199,11 @@ class ShadowCopyAction implements CopyAction { private final Set unused private final ShadowStats stats - private Map visitedFiles = new HashMap() + private Map visitedFiles = new HashMap<>() StreamAction(ZipOutputStream zipOutStr, String encoding, List transformers, - List relocators, PatternSet patternSet, Set unused, - ShadowStats stats) { + List relocators, PatternSet patternSet, Set unused, + ShadowStats stats) { this.zipOutStr = zipOutStr this.transformers = transformers this.relocators = relocators @@ -211,13 +211,21 @@ class ShadowCopyAction implements CopyAction { this.patternSet = patternSet this.unused = unused this.stats = stats - if(encoding != null) { + if (encoding != null) { this.zipOutStr.setEncoding(encoding) } } - private boolean recordVisit(path, size, originJar) { - if (visitedFiles.containsKey(path.toString())) { + /** + * Record visit and return true if visited for the first time. + * + * @param path Visited path. + * @param size Size. + * @param originJar JAR it originated from. + * @return True if wasn't visited already. + */ + private boolean recordVisit(String path, long size, @Nullable RelativePath originJar) { + if (visitedFiles.containsKey(path)) { return false } @@ -296,24 +304,28 @@ class ShadowCopyAction implements CopyAction { } private void visitArchiveFile(ArchiveFileTreeElement archiveFile, ZipFile archive, FileCopyDetails fileDetails) { - def archiveFilePath = archiveFile.relativePath - def archiveFileSize = archiveFile.size + RelativeArchivePath archiveFilePath = archiveFile.relativePath + long archiveFileSize = archiveFile.size if (archiveFile.classFile || !isTransformable(archiveFile)) { - if (recordVisit(archiveFilePath.toString(), archiveFileSize, fileDetails.relativePath) && !isUnused(archiveFilePath.entry.name)) { + def path = archiveFilePath.toString() + if (recordVisit(path, archiveFileSize, archiveFilePath) && !isUnused(archiveFilePath.entry.name)) { if (!remapper.hasRelocators() || !archiveFile.classFile) { copyArchiveEntry(archiveFilePath, archive) } else { remapClass(archiveFilePath, archive) } } else { - def archiveFileInVisitedFiles = visitedFiles.get(archiveFilePath.toString()) + def archiveFileInVisitedFiles = visitedFiles.get(path) if (archiveFileInVisitedFiles && (archiveFileInVisitedFiles.size != fileDetails.size)) { log.warn("IGNORING ${archiveFilePath} from ${fileDetails.relativePath}, size is different (${fileDetails.size} vs ${archiveFileInVisitedFiles.size})") if (archiveFileInVisitedFiles.originJar) { - log.warn(" --> origin JAR was ${archiveFileInVisitedFiles.originJar}") + log.warn("\t--> origin JAR was ${archiveFileInVisitedFiles.originJar}") } else { - log.warn(" --> file originated from project sourcecode") + log.warn("\t--> file originated from project sourcecode") + } + if (new StandardFilesMergeTransformer().canTransformResource(archiveFile)) { + log.warn("\t--> Recommended transformer is " + StandardFilesMergeTransformer.class.name) } } } @@ -408,6 +420,12 @@ class ShadowCopyAction implements CopyAction { } } + /** + * Copy archive entry. + * + * @param archiveFile Source archive entry. + * @param archive Source archive. + */ private void copyArchiveEntry(RelativeArchivePath archiveFile, ZipFile archive) { String mappedPath = remapper.map(archiveFile.entry.name) ZipEntry entry = new ZipEntry(mappedPath) @@ -441,19 +459,20 @@ class ShadowCopyAction implements CopyAction { } private void transform(ArchiveFileTreeElement element, ZipFile archive) { - transformAndClose(element, archive.getInputStream(element.relativePath.entry)) + transformAndClose(element, archive, archive.getInputStream(element.relativePath.entry)) } private void transform(FileCopyDetails details) { - transformAndClose(details, details.file.newInputStream()) + transformAndClose(details, null, details.file.newInputStream()) } - private void transformAndClose(FileTreeElement element, InputStream is) { + private void transformAndClose(FileTreeElement element, @Nullable ZipFile archive, InputStream is) { try { String mappedPath = remapper.map(element.relativePath.pathString) transformers.find { it.canTransformResource(element) }.transform( TransformerContext.builder() .path(mappedPath) + .origin(archive) .is(is) .relocators(relocators) .stats(stats) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java index dc4cfe9e3..00ace7e24 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java @@ -1,29 +1,53 @@ package com.github.jengelman.gradle.plugins.shadow.tasks; +import java.io.File; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; + import com.github.jengelman.gradle.plugins.shadow.ShadowStats; -import com.github.jengelman.gradle.plugins.shadow.internal.*; +import com.github.jengelman.gradle.plugins.shadow.internal.DefaultDependencyFilter; +import com.github.jengelman.gradle.plugins.shadow.internal.DependencyFilter; +import com.github.jengelman.gradle.plugins.shadow.internal.GradleVersionUtil; +import com.github.jengelman.gradle.plugins.shadow.internal.MinimizeDependencyFilter; +import com.github.jengelman.gradle.plugins.shadow.internal.UnusedTracker; +import com.github.jengelman.gradle.plugins.shadow.internal.ZipCompressor; import com.github.jengelman.gradle.plugins.shadow.relocation.CacheableRelocator; import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator; import com.github.jengelman.gradle.plugins.shadow.relocation.SimpleRelocator; -import com.github.jengelman.gradle.plugins.shadow.transformers.*; +import com.github.jengelman.gradle.plugins.shadow.transformers.AppendingTransformer; +import com.github.jengelman.gradle.plugins.shadow.transformers.CacheableTransformer; +import com.github.jengelman.gradle.plugins.shadow.transformers.GroovyExtensionModuleTransformer; +import com.github.jengelman.gradle.plugins.shadow.transformers.ServiceFileTransformer; +import com.github.jengelman.gradle.plugins.shadow.transformers.StandardFilesMergeTransformer; +import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer; import org.gradle.api.Action; +import org.gradle.api.Task; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.DuplicatesStrategy; import org.gradle.api.file.FileCollection; import org.gradle.api.internal.DocumentationRegistry; import org.gradle.api.internal.file.FileResolver; import org.gradle.api.internal.file.copy.CopyAction; +import org.gradle.api.specs.Spec; +import org.gradle.api.tasks.CacheableTask; +import org.gradle.api.tasks.Classpath; +import org.gradle.api.tasks.InputFiles; +import org.gradle.api.tasks.Internal; +import org.gradle.api.tasks.Nested; +import org.gradle.api.tasks.Optional; +import org.gradle.api.tasks.PathSensitive; +import org.gradle.api.tasks.PathSensitivity; +import org.gradle.api.tasks.SourceSet; +import org.gradle.api.tasks.SourceSetContainer; +import org.gradle.api.tasks.TaskAction; import org.gradle.api.tasks.*; import org.gradle.api.tasks.bundling.Jar; import org.gradle.api.tasks.util.PatternSet; import org.jetbrains.annotations.NotNull; -import java.io.File; -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.Callable; - @CacheableTask public class ShadowJar extends Jar implements ShadowSpec { @@ -51,28 +75,49 @@ public FileCollection call() { public ShadowJar() { super(); - setDuplicatesStrategy(DuplicatesStrategy.INCLUDE); //shadow filters out files later. This was the default behavior in Gradle < 6.x + setDuplicatesStrategy( + DuplicatesStrategy.INCLUDE); //shadow filters out files later. This was the default behavior in Gradle < 6.x + versionUtil = new GradleVersionUtil(getProject().getGradle().getGradleVersion()); dependencyFilter = new DefaultDependencyFilter(getProject()); dependencyFilterForMinimize = new MinimizeDependencyFilter(getProject()); - setManifest(new DefaultInheritManifest(getProject(), getServices().get(FileResolver.class))); - transformers = new ArrayList<>(); + setManifest(new DefaultInheritManifest(getServices().get(FileResolver.class))); + /* + Add as default the StandardFilesMergeTransformer, remove it with "removeDefaultTransformers()". + This is added by default, because otherwise: + a) In projects with many dependencies the user gets flooded with information about duplicated entries + like "META-INF/notice.txt", "META-INF/license.txt"... + b) Important licensing information written in META-INF/license.txt and other files may be lost. + c) Helpful information written in readme files may be lost. + d) The merging of plain text files is safe, there is no important logic to follow. Not like MANIFEST.MF, + property files, xml files, etc. Merged HTML may not look that good, but it works. + */ + transformers = new ArrayList<>(Collections.singletonList(new StandardFilesMergeTransformer())); relocators = new ArrayList<>(); configurations = new ArrayList<>(); - this.getInputs().property("minimize", (Callable) () -> minimizeJar); - this.getOutputs().doNotCacheIf("Has one or more transforms or relocators that are not cacheable", task -> { - for (Transformer transformer : transformers) { - if (!isCacheableTransform(transformer.getClass())) { - return true; - } - } - for (Relocator relocator : relocators) { - if (!isCacheableRelocator(relocator.getClass())) { - return true; - } + this.getInputs().property("minimize", new Callable() { + @Override + public Boolean call() throws Exception { + return minimizeJar; } - return false; }); + this.getOutputs().doNotCacheIf("Has one or more transforms or relocators that are not cacheable", + new Spec() { + @Override + public boolean isSatisfiedBy(Task task) { + for (Transformer transformer : transformers) { + if (!isCacheableTransform(transformer.getClass())) { + return true; + } + } + for (Relocator relocator : relocators) { + if (!isCacheableRelocator(relocator.getClass())) { + return true; + } + } + return false; + } + }); } @Override @@ -105,7 +150,8 @@ public InheritManifest getManifest() { @NotNull protected CopyAction createCopyAction() { DocumentationRegistry documentationRegistry = getServices().get(DocumentationRegistry.class); - final UnusedTracker unusedTracker = minimizeJar ? UnusedTracker.forProject(getApiJars(), getSourceSetsClassesDirs().getFiles(), getToMinimize()) : null; + final UnusedTracker unusedTracker = minimizeJar ? UnusedTracker.forProject(getApiJars(), + getSourceSetsClassesDirs().getFiles(), getToMinimize()) : null; return new ShadowCopyAction(getArchiveFile().get().getAsFile(), getInternalCompressor(), documentationRegistry, this.getMetadataCharset(), transformers, relocators, getRootPatternSet(), shadowStats, isPreserveFileTimestamps(), minimizeJar, unusedTracker); @@ -121,7 +167,6 @@ FileCollection getToMinimize() { return toMinimize; } - @Classpath FileCollection getApiJars() { if (apiJars == null) { @@ -132,7 +177,6 @@ FileCollection getApiJars() { return apiJars; } - @InputFiles @PathSensitive(PathSensitivity.RELATIVE) FileCollection getSourceSetsClassesDirs() { @@ -213,12 +257,28 @@ public ShadowJar transform(Class clazz) throws Instantiat * @return this */ @Override - public ShadowJar transform(Class clazz, Action c) throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { + public ShadowJar transform(Class clazz, Action c) + throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { T transformer = clazz.getDeclaredConstructor().newInstance(); addTransform(transformer, c); return this; } + /** + * Removes all default transformers. + *
Right now only {@link StandardFilesMergeTransformer} is added as default transformer, this method removes + * it. + * + * @return this + */ + public ShadowJar removeDefaultTransformers() { + final java.util.Optional standardFilesMergeTransformer = transformers.stream() // + .filter(StandardFilesMergeTransformer.class::isInstance) // + .findAny(); + standardFilesMergeTransformer.ifPresent(transformer -> transformers.remove(transformer)); + return this; + } + private boolean isCacheableTransform(Class clazz) { return clazz.isAnnotationPresent(CacheableTransformer.class); } @@ -340,7 +400,8 @@ public ShadowJar relocate(String pattern, String destination) { */ @Override public ShadowJar relocate(String pattern, String destination, Action configure) { - SimpleRelocator relocator = new SimpleRelocator(pattern, destination, new ArrayList<>(), new ArrayList<>()); + SimpleRelocator relocator = new SimpleRelocator(pattern, destination, new ArrayList(), + new ArrayList()); addRelocator(relocator, configure); return this; } @@ -364,7 +425,8 @@ public ShadowJar relocate(Relocator relocator) { * @return this */ @Override - public ShadowJar relocate(Class relocatorClass) throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { + public ShadowJar relocate(Class relocatorClass) + throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { return relocate(relocatorClass, null); } @@ -384,7 +446,8 @@ private void addRelocator(R relocator, Action configure * @return this */ @Override - public ShadowJar relocate(Class relocatorClass, Action configure) throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { + public ShadowJar relocate(Class relocatorClass, Action configure) + throws InstantiationException, IllegalAccessException, NoSuchMethodException, InvocationTargetException { R relocator = relocatorClass.getDeclaredConstructor().newInstance(); addRelocator(relocator, configure); return this; diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformer.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformer.groovy new file mode 100644 index 000000000..c814dead8 --- /dev/null +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformer.groovy @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License") you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.github.jengelman.gradle.plugins.shadow.transformers + + +import org.apache.commons.io.FilenameUtils +import org.apache.commons.io.IOUtils +import org.apache.tools.zip.ZipEntry +import org.apache.tools.zip.ZipOutputStream +import org.codehaus.plexus.util.IOUtil +import org.gradle.api.file.FileTreeElement + +/** + * Merges standard files, like "META-INF/license.txt", "META-INF/notice.txt", "readme.txt" into one, + * writing as prefix where the content comes from, so no license information or important hints gets lost. + * + * @author Jan-Hendrik Diederich + */ +class StandardFilesMergeTransformer implements Transformer { + private class StandardFile { + List origins = new ArrayList<>(); + String content; + + StandardFile(String origin, String content) { + this.origins.add(origin) + this.content = content + } + } + + private final List mergedFiles = [ + "META-INF/license", // + "META-INF/notice", // + "META-INF/readme", // + "readme", // + ] + + private final List fileExtensions = [ + "txt", "md", "htm", "html" + ] + + // Can't use normal HashMap, ...notice.txt and ...NOTICE.txt would otherwise be different entries. + private Map> fileEntries = new TreeMap<>(String.CASE_INSENSITIVE_ORDER) + + @Override + boolean canTransformResource(FileTreeElement element) { + String path = element.relativePath.pathString + mergedFiles.stream() // + .anyMatch(mergeFile -> { + if (path.equalsIgnoreCase(mergeFile)) { + return true + } else { + for (extension in fileExtensions) { + if (path.equalsIgnoreCase(mergeFile + "." + extension)) { + return true + } + } + return false + } + }) + } + + @Override + void transform(TransformerContext context) { + List files = fileEntries.computeIfAbsent(context.path, key -> new ArrayList<>()) + + OutputStream outputStream = new ByteArrayOutputStream() + IOUtils.copyLarge(context.is, outputStream) + + def fileContent = outputStream.toString() + // Remove leading and trailing newlines. Don't trim whitespaces, so centered headers stay centered. + def trimmedFileContent = fileContent.replaceAll("^[\\r\\n]+|[\\r\\n]+\$", "") + + var standardFile = files.stream() // + .filter(entry -> trimmedFileContent.equalsIgnoreCase(entry.content)) // + .findAny() + String originName = context.origin != null + ? FilenameUtils.getName(context.origin.name) + : "Sourcecode" + if (standardFile.isPresent()) { + standardFile.get().origins.add(originName) + } else { + files.add(new StandardFile(originName, trimmedFileContent)) + } + } + + @Override + boolean hasTransformedResource() { + return fileEntries.size() > 0 + } + + @Override + void modifyOutputStream(ZipOutputStream os, boolean preserveFileTimestamps) { + fileEntries.each { String path, List files -> + ZipEntry entry = new ZipEntry(path) + entry.time = TransformerContext.getEntryTimestamp(preserveFileTimestamps, entry.time) + os.putNextEntry(entry) + IOUtil.copy(toInputStream(files), os) + os.closeEntry() + } + } + + private static InputStream toInputStream(List entries) { + String joined = entries.stream() // + .map(entry -> "Origins: " + entry.origins.sort().join(", ") // + + "\n\n" + entry.content) // + .collect() // + .join("\n" + "=".repeat(80) + "\n") + new ByteArrayInputStream(joined.getBytes()) + } +} \ No newline at end of file diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/TransformerContext.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/TransformerContext.groovy index b951dfa66..d5dfc956b 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/TransformerContext.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/TransformerContext.groovy @@ -5,7 +5,9 @@ import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowCopyAction import groovy.transform.Canonical import groovy.transform.builder.Builder +import org.apache.tools.zip.ZipFile +import javax.annotation.Nullable @Canonical @Builder @@ -16,6 +18,9 @@ class TransformerContext { List relocators ShadowStats stats + @Nullable + ZipFile origin + static long getEntryTimestamp(boolean preserveFileTimestamps, long entryTime) { preserveFileTimestamps ? entryTime : ShadowCopyAction.CONSTANT_TIME_FOR_ZIP_ENTRIES } diff --git a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformerTest.groovy b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformerTest.groovy new file mode 100644 index 000000000..3dba6bf28 --- /dev/null +++ b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformerTest.groovy @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License") you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.github.jengelman.gradle.plugins.shadow.transformers + +import org.junit.Before +import org.junit.Test + +import static org.junit.Assert.assertFalse +import static org.junit.Assert.assertTrue + +/** + * Test for {@link StandardFilesMergeTransformer}. + * + * @author Benjamin Bentmann + * @version $Id: ApacheNoticeResourceTransformerTest.java 673906 2008-07-04 05:03:20Z brett $ + * + * Modified from org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformerTest.java + */ +class StandardFilesMergeTransformerTest extends TransformerTestSupport { + + private StandardFilesMergeTransformer transformer + + static { + /* + * NOTE: The Turkish locale has an usual case transformation for the letters "I" and "i", making it a prime + * choice to test for improper case-less string comparisons. + */ + Locale.setDefault(new Locale("tr")) + } + + @Before + void setUp() { + this.transformer = new StandardFilesMergeTransformer() + } + + @Test + void testCanTransformResource() { + assertTrue(this.transformer.canTransformResource(getFileElement("META-INF/NOTICE"))) + assertTrue(this.transformer.canTransformResource(getFileElement("META-INF/NOTICE.TXT"))) + assertTrue(this.transformer.canTransformResource(getFileElement("META-INF/Notice.txt"))) + assertTrue(this.transformer.canTransformResource(getFileElement("META-INF/Notice.hTml"))) + assertFalse(this.transformer.canTransformResource(getFileElement("META-INF/MANIFEST.MF"))) + } +} \ No newline at end of file From f7a0e4302be364577767cecbd45fe89092215e3a Mon Sep 17 00:00:00 2001 From: Another User Date: Fri, 10 Jun 2022 14:51:50 +0200 Subject: [PATCH 05/18] Made StandardFilesMergeTransformer cacheable. --- .../shadow/transformers/StandardFilesMergeTransformer.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformer.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformer.groovy index c814dead8..72c1639d2 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformer.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/StandardFilesMergeTransformer.groovy @@ -33,8 +33,9 @@ import org.gradle.api.file.FileTreeElement * * @author Jan-Hendrik Diederich */ +@CacheableTransformer class StandardFilesMergeTransformer implements Transformer { - private class StandardFile { + private class StandardFile implements Serializable { List origins = new ArrayList<>(); String content; From cb49011384a397ed52a54b52089ff217745d594d Mon Sep 17 00:00:00 2001 From: Jonathan Chapman and Jim Cifarelli Date: Mon, 16 Mar 2015 11:23:43 -0400 Subject: [PATCH 06/18] Displaying `warn` level messages when adding a file attempts to overwrite an existing resource --- .../shadow/tasks/ShadowCopyAction.groovy | 34 +++++-------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy index 5a2585a1c..a4af88032 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy @@ -199,7 +199,7 @@ class ShadowCopyAction implements CopyAction { private final Set unused private final ShadowStats stats - private Map visitedFiles = new HashMap<>() + private Map visitedFiles = new HashMap() StreamAction(ZipOutputStream zipOutStr, String encoding, List transformers, List relocators, PatternSet patternSet, Set unused, @@ -216,16 +216,8 @@ class ShadowCopyAction implements CopyAction { } } - /** - * Record visit and return true if visited for the first time. - * - * @param path Visited path. - * @param size Size. - * @param originJar JAR it originated from. - * @return True if wasn't visited already. - */ - private boolean recordVisit(String path, long size, @Nullable RelativePath originJar) { - if (visitedFiles.containsKey(path)) { + private boolean recordVisit(path, size, originJar) { + if (visitedFiles.containsKey(path.toString())) { return false } @@ -304,29 +296,21 @@ class ShadowCopyAction implements CopyAction { } private void visitArchiveFile(ArchiveFileTreeElement archiveFile, ZipFile archive, FileCopyDetails fileDetails) { - RelativeArchivePath archiveFilePath = archiveFile.relativePath - long archiveFileSize = archiveFile.size + def archiveFilePath = archiveFile.relativePath + def archiveFileSize = archiveFile.size if (archiveFile.classFile || !isTransformable(archiveFile)) { - def path = archiveFilePath.toString() - if (recordVisit(path, archiveFileSize, archiveFilePath) && !isUnused(archiveFilePath.entry.name)) { + if (recordVisit(archiveFilePath.toString(), archiveFileSize, fileDetails.relativePath) && !isUnused(archiveFilePath.entry.name)) { if (!remapper.hasRelocators() || !archiveFile.classFile) { copyArchiveEntry(archiveFilePath, archive) } else { remapClass(archiveFilePath, archive) } } else { - def archiveFileInVisitedFiles = visitedFiles.get(path) - if (archiveFileInVisitedFiles && (archiveFileInVisitedFiles.size != fileDetails.size)) { + def archiveFileInVisitedFiles = visitedFiles.get(archiveFilePath.toString()) + if (archiveFileInVisitedFiles && (archiveFileInVisitedFiles.size != fileDetails.size) && !archiveFilePath.toString().startsWith('META-INF/')) { log.warn("IGNORING ${archiveFilePath} from ${fileDetails.relativePath}, size is different (${fileDetails.size} vs ${archiveFileInVisitedFiles.size})") - if (archiveFileInVisitedFiles.originJar) { - log.warn("\t--> origin JAR was ${archiveFileInVisitedFiles.originJar}") - } else { - log.warn("\t--> file originated from project sourcecode") - } - if (new StandardFilesMergeTransformer().canTransformResource(archiveFile)) { - log.warn("\t--> Recommended transformer is " + StandardFilesMergeTransformer.class.name) - } + log.warn(" --> origin JAR was ${archiveFileInVisitedFiles.originJar}") } } } else { From 5dca0369f692f3ef0b173e77695d9613c92001a8 Mon Sep 17 00:00:00 2001 From: Jonathan Chapman and Jim Cifarelli Date: Thu, 19 Mar 2015 09:56:29 -0400 Subject: [PATCH 07/18] Adding tests, not filtering META-INF/ by default --- .../gradle/plugins/shadow/tasks/ShadowCopyAction.groovy | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy index a4af88032..5559387a1 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy @@ -308,9 +308,13 @@ class ShadowCopyAction implements CopyAction { } } else { def archiveFileInVisitedFiles = visitedFiles.get(archiveFilePath.toString()) - if (archiveFileInVisitedFiles && (archiveFileInVisitedFiles.size != fileDetails.size) && !archiveFilePath.toString().startsWith('META-INF/')) { + if (archiveFileInVisitedFiles && (archiveFileInVisitedFiles.size != fileDetails.size)) { log.warn("IGNORING ${archiveFilePath} from ${fileDetails.relativePath}, size is different (${fileDetails.size} vs ${archiveFileInVisitedFiles.size})") - log.warn(" --> origin JAR was ${archiveFileInVisitedFiles.originJar}") + if (archiveFileInVisitedFiles.originJar) { + log.warn(" --> origin JAR was ${archiveFileInVisitedFiles.originJar}") + } else { + log.warn(" --> file originated from project sourcecode") + } } } } else { From 104f31c2a32ca66383b76f7479e2ab73ed8f004b Mon Sep 17 00:00:00 2001 From: Another User Date: Fri, 10 Jun 2022 12:37:09 +0200 Subject: [PATCH 08/18] Cleaned up "Collision logging" and added StandardFilesMergeTransformer to merge standard files --- .../shadow/tasks/ShadowCopyAction.groovy | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy index 5559387a1..cfd310d16 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy @@ -199,7 +199,7 @@ class ShadowCopyAction implements CopyAction { private final Set unused private final ShadowStats stats - private Map visitedFiles = new HashMap() + private Map visitedFiles = new HashMap<>() StreamAction(ZipOutputStream zipOutStr, String encoding, List transformers, List relocators, PatternSet patternSet, Set unused, @@ -216,13 +216,21 @@ class ShadowCopyAction implements CopyAction { } } - private boolean recordVisit(path, size, originJar) { - if (visitedFiles.containsKey(path.toString())) { + /** + * Record visit and return true if visited for the first time. + * + * @param path Visited path. + * @param size Size. + * @param originJar JAR it originated from. + * @return True if wasn't visited already. + */ + private boolean recordVisit(String path, long size, @Nullable RelativePath originJar) { + if (visitedFiles.containsKey(path)) { return false } if (originJar == null) { - originJar = "" + originJar = new RelativePath(false) } visitedFiles.put(path.toString(), [size: size, originJar: originJar]) @@ -296,24 +304,28 @@ class ShadowCopyAction implements CopyAction { } private void visitArchiveFile(ArchiveFileTreeElement archiveFile, ZipFile archive, FileCopyDetails fileDetails) { - def archiveFilePath = archiveFile.relativePath - def archiveFileSize = archiveFile.size + RelativeArchivePath archiveFilePath = archiveFile.relativePath + long archiveFileSize = archiveFile.size if (archiveFile.classFile || !isTransformable(archiveFile)) { - if (recordVisit(archiveFilePath.toString(), archiveFileSize, fileDetails.relativePath) && !isUnused(archiveFilePath.entry.name)) { + def path = archiveFilePath.toString() + if (recordVisit(path, archiveFileSize, archiveFilePath) && !isUnused(archiveFilePath.entry.name)) { if (!remapper.hasRelocators() || !archiveFile.classFile) { copyArchiveEntry(archiveFilePath, archive) } else { remapClass(archiveFilePath, archive) } } else { - def archiveFileInVisitedFiles = visitedFiles.get(archiveFilePath.toString()) + def archiveFileInVisitedFiles = visitedFiles.get(path) if (archiveFileInVisitedFiles && (archiveFileInVisitedFiles.size != fileDetails.size)) { log.warn("IGNORING ${archiveFilePath} from ${fileDetails.relativePath}, size is different (${fileDetails.size} vs ${archiveFileInVisitedFiles.size})") if (archiveFileInVisitedFiles.originJar) { - log.warn(" --> origin JAR was ${archiveFileInVisitedFiles.originJar}") + log.warn("\t--> origin JAR was ${archiveFileInVisitedFiles.originJar}") } else { - log.warn(" --> file originated from project sourcecode") + log.warn("\t--> file originated from project sourcecode") + } + if (new StandardFilesMergeTransformer().canTransformResource(archiveFile)) { + log.warn("\t--> Recommended transformer is " + StandardFilesMergeTransformer.class.name) } } } From cd774515cff7d7022f755cbdbebf1dccdbbfb574 Mon Sep 17 00:00:00 2001 From: Jan-Hendrik Diederich Date: Fri, 3 Mar 2023 12:30:55 +0100 Subject: [PATCH 09/18] Fixed merge problems --- .../jengelman/gradle/plugins/shadow/tasks/ShadowJar.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java index 00ace7e24..806a0f8dd 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java @@ -7,11 +7,14 @@ import java.util.List; import java.util.concurrent.Callable; +import javax.annotation.Nonnull; + import com.github.jengelman.gradle.plugins.shadow.ShadowStats; import com.github.jengelman.gradle.plugins.shadow.internal.DefaultDependencyFilter; import com.github.jengelman.gradle.plugins.shadow.internal.DependencyFilter; import com.github.jengelman.gradle.plugins.shadow.internal.GradleVersionUtil; import com.github.jengelman.gradle.plugins.shadow.internal.MinimizeDependencyFilter; +import com.github.jengelman.gradle.plugins.shadow.internal.RelocationUtil; import com.github.jengelman.gradle.plugins.shadow.internal.UnusedTracker; import com.github.jengelman.gradle.plugins.shadow.internal.ZipCompressor; import com.github.jengelman.gradle.plugins.shadow.relocation.CacheableRelocator; @@ -24,16 +27,15 @@ import com.github.jengelman.gradle.plugins.shadow.transformers.StandardFilesMergeTransformer; import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer; import org.gradle.api.Action; -import org.gradle.api.Task; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.DuplicatesStrategy; import org.gradle.api.file.FileCollection; import org.gradle.api.internal.DocumentationRegistry; import org.gradle.api.internal.file.FileResolver; import org.gradle.api.internal.file.copy.CopyAction; -import org.gradle.api.specs.Spec; import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.Classpath; +import org.gradle.api.tasks.Input; import org.gradle.api.tasks.InputFiles; import org.gradle.api.tasks.Internal; import org.gradle.api.tasks.Nested; @@ -46,7 +48,6 @@ import org.gradle.api.tasks.*; import org.gradle.api.tasks.bundling.Jar; import org.gradle.api.tasks.util.PatternSet; -import org.jetbrains.annotations.NotNull; @CacheableTask public class ShadowJar extends Jar implements ShadowSpec { @@ -147,7 +148,7 @@ public InheritManifest getManifest() { } @Override - @NotNull + @Nonnull protected CopyAction createCopyAction() { DocumentationRegistry documentationRegistry = getServices().get(DocumentationRegistry.class); final UnusedTracker unusedTracker = minimizeJar ? UnusedTracker.forProject(getApiJars(), From abb9ac3775684546ca5dea04a8cdbe87028c844c Mon Sep 17 00:00:00 2001 From: Jan Diederich Date: Tue, 31 Oct 2023 20:06:52 +0100 Subject: [PATCH 10/18] Lower the log-level for duplicated "META-INF/MANIFEST.MF" from warn to debug So the user doesn't get flooded with superfluous logs --- .../shadow/tasks/ShadowCopyAction.groovy | 29 +++++++++++++++--- .../plugins/shadow/ShadowPluginSpec.groovy | 6 ++-- .../resources/test-artifact-1.0-SNAPSHOT.jar | Bin 3115 -> 3324 bytes .../resources/test-project-1.0-SNAPSHOT.jar | Bin 3906 -> 4123 bytes 4 files changed, 26 insertions(+), 9 deletions(-) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy index cfd310d16..1a2bdbb94 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy @@ -8,6 +8,7 @@ import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator import com.github.jengelman.gradle.plugins.shadow.transformers.StandardFilesMergeTransformer import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext +import groovy.util.logging.Log import groovy.util.logging.Slf4j import org.apache.commons.io.FilenameUtils import org.apache.commons.io.IOUtils @@ -16,6 +17,7 @@ import org.apache.tools.zip.Zip64RequiredException import org.apache.tools.zip.ZipEntry import org.apache.tools.zip.ZipFile import org.apache.tools.zip.ZipOutputStream +import org.codehaus.groovy.transform.LogASTTransformation import org.gradle.api.Action import org.gradle.api.GradleException import org.gradle.api.UncheckedIOException @@ -38,14 +40,18 @@ import org.objectweb.asm.ClassReader import org.objectweb.asm.ClassVisitor import org.objectweb.asm.ClassWriter import org.objectweb.asm.commons.ClassRemapper +import org.slf4j.Logger +import org.slf4j.LoggerFactory import javax.annotation.Nullable import java.util.zip.ZipException -@Slf4j + class ShadowCopyAction implements CopyAction { static final long CONSTANT_TIME_FOR_ZIP_ENTRIES = (new GregorianCalendar(1980, 1, 1, 0, 0, 0)).getTimeInMillis() + final static Logger log = LoggerFactory.getLogger(ShadowCopyAction.class); + private final File zipFile private final ZipCompressor compressor private final DocumentationRegistry documentationRegistry @@ -318,14 +324,27 @@ class ShadowCopyAction implements CopyAction { } else { def archiveFileInVisitedFiles = visitedFiles.get(path) if (archiveFileInVisitedFiles && (archiveFileInVisitedFiles.size != fileDetails.size)) { - log.warn("IGNORING ${archiveFilePath} from ${fileDetails.relativePath}, size is different (${fileDetails.size} vs ${archiveFileInVisitedFiles.size})") + // Give of only a debug-level warning for this file: + final String lowLevelWarningFile = "META-INF/MANIFEST.MF" + + final logDebug = (String msg) -> { log.debug(msg) } + final logWarn = (String msg) -> { log.warn(msg) } + + final Closure logger + if (archiveFilePath.toString() == lowLevelWarningFile) { + logger = logDebug + } else { + logger = logWarn + } + logger("IGNORING ${archiveFilePath} from ${fileDetails.relativePath}," + + " size is different (${fileDetails.size} vs ${archiveFileInVisitedFiles.size})") if (archiveFileInVisitedFiles.originJar) { - log.warn("\t--> origin JAR was ${archiveFileInVisitedFiles.originJar}") + logger("\t--> origin JAR was ${archiveFileInVisitedFiles.originJar}") } else { - log.warn("\t--> file originated from project sourcecode") + logger("\t--> file originated from project sourcecode") } if (new StandardFilesMergeTransformer().canTransformResource(archiveFile)) { - log.warn("\t--> Recommended transformer is " + StandardFilesMergeTransformer.class.name) + logger("\t--> Recommended transformer is " + StandardFilesMergeTransformer.class.name) } } } diff --git a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy index 9f74f22d8..fa120bb04 100644 --- a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy +++ b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy @@ -151,10 +151,8 @@ class ShadowPluginSpec extends PluginSpecification { ExecutionResult result = runner.run() then: - success(result) - assert result.standardOutput =~ /IGNORING META-INF\/MANIFEST\.MF from test-artifact-1\.0-SNAPSHOT\.jar, size is different \(3115 vs 25\)\s --> file originated from project sourcecode/ - assert result.standardOutput =~ /IGNORING META-INF\/MANIFEST\.MF from test-project-1\.0-SNAPSHOT\.jar, size is different \(3906 vs 25\)\s --> file originated from project sourcecode/ - } + assert result.output =~ /\s*IGNORING Weird-File\.StrangeFormat from test-project-1\.0-SNAPSHOT\.jar, size is different \([0-9]{4} vs [0-9]{2}\)\s+--> origin JAR was Weird-File.StrangeFormat/ + } def 'include project sources'() { given: diff --git a/src/test/resources/test-artifact-1.0-SNAPSHOT.jar b/src/test/resources/test-artifact-1.0-SNAPSHOT.jar index 009abad49de5cbf4857e8c429cb1f81c75c2dd56..639ac32a8e40d6b3be6ad80efb0ded1cb45a8eb6 100644 GIT binary patch delta 259 zcmZ22@kerl97jDf3l{?jT$>snKKIR1CPg3{ge4ec7{XICi&AvmGILV(f=h}L^U_n@ z@{4j4OF~098JJsa9aBNLw1S&~k>v$50|S^q=utq?qmZ1RSCX1n!WH1n$RxrHv9zpV zG9M@3W($t{%tee$a!^rBr$~TY$iN81TN-6J81OiZ72+^7M+JDZvVlzC1Hy0?28Mp1 H84L^np!z;i delta 33 jcmew(xmsd_9LMH~96y;wSlJkWfENgVGBYs9aDaFKnR^CM diff --git a/src/test/resources/test-project-1.0-SNAPSHOT.jar b/src/test/resources/test-project-1.0-SNAPSHOT.jar index f80e03f9031a76572f862ed88eed6b8d722ba947..84824582d222eba0ec156ed6cb92df519d7a7322 100644 GIT binary patch delta 259 zcmX>kH(OzY40k;<3l{?jJenFG{$%#PKusVUge4ec7{XICi&AvmGILV(f=h}L^U_n@ z@{4j4OF~098JMTrI;MhfX$3a}Bg+eB1_m(UmtUfgs89~lsgRtXSCX1nqL7&f6e@<8 zppXVOLXRuJn~_O`dGdE&vCU@OKbVUcndF#pIYk2GLIy@4-qI+=&49;YtPqExIV!-L Ul?`M9KM)qMFfdF7n!&&T08scr!T04EN+bKB>(cxqmQ=u(B}#0Ur Date: Wed, 1 Nov 2023 18:00:01 +0100 Subject: [PATCH 11/18] Adding JsonTransformer and tests for it --- .../transformers/JsonTransformer.groovy | 158 ++++++++++++++++++ .../JsonAppendingTransformerTest.groovy | 158 ++++++++++++++++++ .../resources/test-artifact-1.0-SNAPSHOT.jar | Bin 3324 -> 3725 bytes .../resources/test-project-1.0-SNAPSHOT.jar | Bin 4123 -> 4541 bytes 4 files changed, 316 insertions(+) create mode 100644 src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/JsonTransformer.groovy create mode 100644 src/test/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/JsonAppendingTransformerTest.groovy diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/JsonTransformer.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/JsonTransformer.groovy new file mode 100644 index 000000000..810a34988 --- /dev/null +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/JsonTransformer.groovy @@ -0,0 +1,158 @@ +package com.github.jengelman.gradle.plugins.shadow.transformers + +import org.gradle.api.file.FileTreeElement +import org.gradle.api.logging.Logging +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.Optional + +import com.google.gson.JsonArray +import com.google.gson.JsonNull +import com.google.gson.JsonObject +import com.google.gson.JsonElement +import com.google.gson.JsonPrimitive +import com.google.gson.JsonParser +import com.google.gson.Gson + +import org.apache.tools.zip.ZipOutputStream +import org.apache.tools.zip.ZipEntry + +/** + * Merge multiple occurrence of JSON files. + * + * @author Logic Fan, extended to process an array of files by Jan-Hendrik Diederich + */ +@CacheableTransformer +class JsonTransformer implements Transformer { + private static final GSON = new Gson() + private static final LOGGER = Logging.getLogger(JsonTransformer.class) + + @Optional + @Input + List paths + + private Map matchedPath = [:] + + @Override + boolean canTransformResource(FileTreeElement element) { + String path = element.relativePath.pathString + for (p in paths) { + if (path.equalsIgnoreCase(p)) { + matchedPath[path] = null + return true + } + } + return false + } + + @Override + void transform(TransformerContext context) { + String path = context.getPath() + final JsonElement j + try { + j = JsonParser.parseReader(new InputStreamReader(context.is, "UTF-8")) + } catch (Exception e) { + throw new RuntimeException("error on processing json", e) + } + + matchedPath[path] = (matchedPath[path] == null) ? j : mergeJson(matchedPath[path], j) + } + + @Override + boolean hasTransformedResource() { + return !matchedPath.isEmpty() + } + + @Override + void modifyOutputStream(ZipOutputStream os, boolean preserveFileTimestamps) { + if (paths == null) { + throw new IllegalArgumentException("\"paths\" is null and not set") + } + for (Map.Entry entrySet in matchedPath) { + ZipEntry entry = new ZipEntry(entrySet.key) + entry.time = TransformerContext.getEntryTimestamp(preserveFileTimestamps, entry.time) + os.putNextEntry(entry) + os.write(GSON.toJson(entrySet.value).getBytes()) + } + matchedPath = [:] + } + + /** + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
{@code lhs} {@code rhs} {@code return}
Any {@code JsonNull} {@code lhs}
{@code JsonNull} Any {@code rhs}
{@code JsonArray} {@code JsonArray} concatenation
{@code JsonObject} {@code JsonObject} merge for each key
{@code JsonPrimitive} {@code JsonPrimitive}return lhs if {@code lhs.equals(rhs)}, error otherwise
Other error
+ * @param lhs a {@code JsonElement} + * @param rhs a {@code JsonElement} + * @param id used for logging purpose only + * @return the merged {@code JsonElement} + */ + private static JsonElement mergeJson(JsonElement lhs, JsonElement rhs, String id = "") { + if (rhs == null || rhs instanceof JsonNull) { + return lhs + } else if (lhs == null || lhs instanceof JsonNull) { + return rhs + } else if (lhs instanceof JsonArray && rhs instanceof JsonArray) { + return mergeJsonArray(lhs as JsonArray, rhs as JsonArray) + } else if (lhs instanceof JsonObject && rhs instanceof JsonObject) { + return mergeJsonObject(lhs as JsonObject, rhs as JsonObject, id) + } else if (lhs instanceof JsonPrimitive && rhs instanceof JsonPrimitive) { + return mergeJsonPrimitive(lhs as JsonPrimitive, rhs as JsonPrimitive, id) + } else { + LOGGER.warn("conflicts for property {} detected, {} & {}", + id, lhs.toString(), rhs.toString()) + return lhs + } + } + + private static JsonPrimitive mergeJsonPrimitive(JsonPrimitive lhs, JsonPrimitive rhs, String id) { + // In Groovy, {@code a == b} is equivalent to {@code a.equals(b)} + if (lhs != rhs) { + LOGGER.warn("conflicts for property {} detected, {} & {}", + id, lhs.toString(), rhs.toString()) + } + return lhs + } + + private static JsonObject mergeJsonObject(JsonObject lhs, JsonObject rhs, String id) { + JsonObject object = new JsonObject() + + Set properties = new HashSet<>() + properties.addAll(lhs.keySet()) + properties.addAll(rhs.keySet()) + for (String property : properties) { + object.add(property, + mergeJson(lhs.get(property), rhs.get(property), id + ":" + property)) + } + + return object + } + + private static JsonArray mergeJsonArray(JsonArray lhs, JsonArray rhs) { + JsonArray array = new JsonArray() + + array.addAll(lhs) + array.addAll(rhs) + + return array + } +} \ No newline at end of file diff --git a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/JsonAppendingTransformerTest.groovy b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/JsonAppendingTransformerTest.groovy new file mode 100644 index 000000000..f6d373af2 --- /dev/null +++ b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/transformers/JsonAppendingTransformerTest.groovy @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License") you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.github.jengelman.gradle.plugins.shadow.transformers + +import com.github.jengelman.gradle.plugins.shadow.ShadowStats +import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator +import com.google.gson.JsonArray +import com.google.gson.JsonElement +import com.google.gson.JsonObject +import com.google.gson.JsonParser +import org.apache.tools.zip.ZipOutputStream +import org.junit.Before +import org.junit.Test + +import java.util.zip.ZipEntry +import java.util.zip.ZipFile +import java.util.zip.ZipInputStream + +import static org.junit.Assert.assertEquals +import static org.junit.Assert.assertFalse +import static org.junit.Assert.assertTrue + +/** + * Test for {@link JsonTransformer}. + * + * @author Jan-Hendrik Diederich + * + * Modified from com.github.jengelman.gradle.plugins.shadow.transformers.XmlAppendingTransformerTest.java + */ +class JsonAppendingTransformerTest extends TransformerTestSupport { + + JsonTransformer transformer + + static final String TEST_ARTIFACT_JAR = 'test-artifact-1.0-SNAPSHOT.jar' + static final String TEST_PROJECT_JAR = 'test-project-1.0-SNAPSHOT.jar' + + static final String TEST_JSON = 'test.json' + static final String TEST2_JSON = 'test2.json' + + @Before + void setUp() { + transformer = new JsonTransformer() + } + + @Test + void testCanTransformResource() { + transformer.paths = ["test.json"] + + assertTrue(this.transformer.canTransformResource(getFileElement("test.json"))) + assertFalse(this.transformer.canTransformResource(getFileElement("META-INF/MANIFEST.MF"))) + } + + @Test + void transformResource() { + transformer.transform(new TransformerContext(TEST_JSON, readFromTestJar(TEST_ARTIFACT_JAR, TEST_JSON), + Collections. emptyList(), new ShadowStats())) + transformer.transform(new TransformerContext(TEST2_JSON, readFromTestJar(TEST_ARTIFACT_JAR, TEST2_JSON), + Collections. emptyList(), new ShadowStats())) + + transformer.transform(new TransformerContext(TEST_JSON, readFromTestJar(TEST_PROJECT_JAR, TEST_JSON), + Collections. emptyList(), new ShadowStats())) + transformer.transform(new TransformerContext(TEST2_JSON, readFromTestJar(TEST_PROJECT_JAR, TEST2_JSON), + Collections. emptyList(), new ShadowStats())) + + def zipFileName = "testable-zip-file-" + def zipFileSuffix = ".jar" + def testableZipFile = File.createTempFile(zipFileName, zipFileSuffix) + def fileOutputStream = new FileOutputStream(testableZipFile) + def bufferedOutputStream = new BufferedOutputStream(fileOutputStream) + def zipOutputStream = new ZipOutputStream(bufferedOutputStream) + + transformer.paths = [TEST_JSON, TEST2_JSON] + try { + transformer.modifyOutputStream(zipOutputStream, false) + } finally { + zipOutputStream.close() + bufferedOutputStream.close() + fileOutputStream.close() + } + // Read 1st file. + String targetJson = readFromZipFile(testableZipFile.absolutePath, TEST_JSON) + println("Target JSON: \"" + targetJson + "\"") + + assertFalse(targetJson.isEmpty()) + assertTrue(targetJson.contains("\"C: Only here\"")) + + JsonElement jsonElement = JsonParser.parseString(targetJson) + JsonObject jsonObject = jsonElement.getAsJsonObject() + + JsonElement subAA = jsonObject.get("a.a") + + JsonElement subAA1 = subAA.getAsJsonObject().get("a.sub1") + assertEquals("A Sub 1", subAA1.asString) + + JsonElement subAA2 = subAA.getAsJsonObject().get("a.sub2") + assertEquals("A Sub 2", subAA2.asString) + + // Read 2nd file. + String target2Json = readFromZipFile(testableZipFile.absolutePath, TEST2_JSON) + assertFalse(target2Json.isEmpty()) + JsonElement jsonElement2 = JsonParser.parseString(target2Json) + JsonObject jsonObject2 = jsonElement2.getAsJsonObject() + JsonArray jsonArray2 = jsonObject2.get("Array").asJsonArray + assertEquals(List.of("A", "B", "C", "C", "D", "E"), + (List) jsonArray2.collect({ it -> it.getAsString() })) + } + + static InputStream readFromTestJar(String resourceName, String fileName) { + try (ZipInputStream inputStream = new ZipInputStream(getResourceStream(resourceName))) { + while (true) { + ZipEntry entry = inputStream.nextEntry + if (entry == null) { + break + } else if (entry.name == fileName) { + // Read the content of the entry + byte[] buffer = new byte[entry.size] + inputStream.read(buffer) + return new ByteArrayInputStream(buffer) + } + } + } + throw new IllegalArgumentException("Missing entry " + fileName) + } + + static String readFromZipFile(String resourceName, String fileName) { + def zip = new ZipFile(resourceName) + try { + ZipEntry entry = zip.getEntry(fileName) + if (!entry) { + throw new IllegalArgumentException("Missing entry " + fileName + " in " + resourceName) + } + return new String(zip.getInputStream(entry).readAllBytes()) + } finally { + zip.close() + } + } + + private static InputStream getResourceStream(String resource) { + JsonAppendingTransformerTest.class.classLoader.getResourceAsStream(resource) + } +} diff --git a/src/test/resources/test-artifact-1.0-SNAPSHOT.jar b/src/test/resources/test-artifact-1.0-SNAPSHOT.jar index 639ac32a8e40d6b3be6ad80efb0ded1cb45a8eb6..dbe60ba230a033e51668b8fa924b9f141bf62d1f 100644 GIT binary patch delta 449 zcmew(*(UaN>YnU^sC$r#Oh~30RdhiK^vGRwRu@3%#)iD&zsU*d332~TLNnz*TJQp zTy0(solzT3scbL^n6_GsA;24EDPLoKVmR}C87)hojm|*K1+~=(VeRWor&OjaC+yaq z*3r@n+!Pe>>AZL7CB{yc zZD(W>Vder_#=ub4uz4XyV2}W-WCW618v8jIkOPGk5-4Z^0=Y@uFGB6*W z=a||8#HAJ742&!C$r#Oh~30RdhiK^vGRwRu@3%#)iD&zsU*d333#TLNnz*TJQp zTy98+%)pS)^R#kCMIbneN8{8F2@@Px65u(B}#0Y4BHurM%8 Date: Fri, 3 Nov 2023 12:36:36 +0100 Subject: [PATCH 12/18] Logging module-info.class collisions --- .../shadow/tasks/ShadowCopyAction.groovy | 38 +++++++++++++++++-- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy index 1a2bdbb94..b8b475496 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy @@ -8,8 +8,6 @@ import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator import com.github.jengelman.gradle.plugins.shadow.transformers.StandardFilesMergeTransformer import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext -import groovy.util.logging.Log -import groovy.util.logging.Slf4j import org.apache.commons.io.FilenameUtils import org.apache.commons.io.IOUtils import org.apache.tools.zip.UnixStat @@ -17,7 +15,6 @@ import org.apache.tools.zip.Zip64RequiredException import org.apache.tools.zip.ZipEntry import org.apache.tools.zip.ZipFile import org.apache.tools.zip.ZipOutputStream -import org.codehaus.groovy.transform.LogASTTransformation import org.gradle.api.Action import org.gradle.api.GradleException import org.gradle.api.UncheckedIOException @@ -314,7 +311,40 @@ class ShadowCopyAction implements CopyAction { long archiveFileSize = archiveFile.size if (archiveFile.classFile || !isTransformable(archiveFile)) { - def path = archiveFilePath.toString() + String path = archiveFilePath.toString() + + if (path.endsWith("module-info.class")) { + log.warn("module-info collision") + + def moduleFileName = "module-info" + def moduleFileSuffix = ".class" + File disassembleModFile = File.createTempFile(moduleFileName, moduleFileSuffix) + + try (InputStream is = archive.getInputStream(archiveFilePath.entry)) { + try (OutputStream os = new FileOutputStream(disassembleModFile)) { + IOUtils.copyLarge(is, os) + } + } + + ProcessBuilder processBuilder = new ProcessBuilder("javap", disassembleModFile.absolutePath) + processBuilder.redirectErrorStream(true) + Process process = processBuilder.start() + InputStream inputStream = process.getInputStream() + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)) + String line + + while ((line = reader.readLine()) != null) { + log.warn(line) + } + + int exitCode = process.waitFor() + if (exitCode != 0) { + log.warn("Process exited with code " + exitCode) + } + + log.warn("module-info collision end") + } + if (recordVisit(path, archiveFileSize, archiveFilePath) && !isUnused(archiveFilePath.entry.name)) { if (!remapper.hasRelocators() || !archiveFile.classFile) { copyArchiveEntry(archiveFilePath, archive) From 2fc05a93a616c2838d1372c7b0230c6c7f8bd3ce Mon Sep 17 00:00:00 2001 From: Jan Diederich Date: Wed, 4 Sep 2024 06:47:37 +0200 Subject: [PATCH 13/18] Fix rebase errors --- .../shadow/tasks/ShadowCopyAction.groovy | 9 ++--- .../plugins/shadow/tasks/ShadowJar.java | 33 ++++++++----------- .../plugins/shadow/ShadowPluginSpec.groovy | 7 ++-- 3 files changed, 21 insertions(+), 28 deletions(-) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy index b8b475496..b564efbc1 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy @@ -19,10 +19,12 @@ import org.gradle.api.Action import org.gradle.api.GradleException import org.gradle.api.UncheckedIOException import org.gradle.api.file.FileCopyDetails +import org.gradle.api.file.FilePermissions import org.gradle.api.file.FileTreeElement import org.gradle.api.file.RelativePath import org.gradle.api.internal.DocumentationRegistry import org.gradle.api.internal.file.CopyActionProcessingStreamAction +import org.gradle.api.internal.file.DefaultFilePermissions import org.gradle.api.internal.file.DefaultFileTreeElement import org.gradle.api.internal.file.copy.CopyAction import org.gradle.api.internal.file.copy.CopyActionProcessingStream @@ -43,7 +45,6 @@ import org.slf4j.LoggerFactory import javax.annotation.Nullable import java.util.zip.ZipException - class ShadowCopyAction implements CopyAction { static final long CONSTANT_TIME_FOR_ZIP_ENTRIES = (new GregorianCalendar(1980, 1, 1, 0, 0, 0)).getTimeInMillis() @@ -245,7 +246,7 @@ class ShadowCopyAction implements CopyAction { } private boolean recordVisit(FileCopyDetails fileCopyDetails) { - return recordVisit(fileCopyDetails.relativePath, fileCopyDetails.size, null) + return recordVisit(fileCopyDetails.relativePath.toString(), fileCopyDetails.size, null) } @Override @@ -282,11 +283,11 @@ class ShadowCopyAction implements CopyAction { ZipFile archive = new ZipFile(fileDetails.file) try { List archiveElements = archive.entries.collect { - new ArchiveFileTreeElement(new RelativeArchivePath(it, fileDetails)) + new ArchiveFileTreeElement(new RelativeArchivePath(it)) } Spec patternSpec = patternSet.getAsSpec() List filteredArchiveElements = archiveElements.findAll { ArchiveFileTreeElement archiveElement -> - patternSpec.isSatisfiedBy(archiveElement) + patternSpec.isSatisfiedBy(archiveElement.asFileTreeElement()) } filteredArchiveElements.each { ArchiveFileTreeElement archiveElement -> if (archiveElement.relativePath.file) { diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java index 806a0f8dd..6065845f0 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java @@ -27,12 +27,14 @@ import com.github.jengelman.gradle.plugins.shadow.transformers.StandardFilesMergeTransformer; import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer; import org.gradle.api.Action; +import org.gradle.api.Task; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.DuplicatesStrategy; import org.gradle.api.file.FileCollection; import org.gradle.api.internal.DocumentationRegistry; import org.gradle.api.internal.file.FileResolver; import org.gradle.api.internal.file.copy.CopyAction; +import org.gradle.api.specs.Spec; import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.Input; @@ -78,10 +80,9 @@ public ShadowJar() { super(); setDuplicatesStrategy( DuplicatesStrategy.INCLUDE); //shadow filters out files later. This was the default behavior in Gradle < 6.x - versionUtil = new GradleVersionUtil(getProject().getGradle().getGradleVersion()); dependencyFilter = new DefaultDependencyFilter(getProject()); dependencyFilterForMinimize = new MinimizeDependencyFilter(getProject()); - setManifest(new DefaultInheritManifest(getServices().get(FileResolver.class))); + setManifest(new DefaultInheritManifest(getProject(), getServices().get(FileResolver.class))); /* Add as default the StandardFilesMergeTransformer, remove it with "removeDefaultTransformers()". This is added by default, because otherwise: @@ -96,28 +97,20 @@ public ShadowJar() { relocators = new ArrayList<>(); configurations = new ArrayList<>(); - this.getInputs().property("minimize", new Callable() { - @Override - public Boolean call() throws Exception { - return minimizeJar; - } - }); + this.getInputs().property("minimize", (Callable) () -> minimizeJar); this.getOutputs().doNotCacheIf("Has one or more transforms or relocators that are not cacheable", - new Spec() { - @Override - public boolean isSatisfiedBy(Task task) { - for (Transformer transformer : transformers) { - if (!isCacheableTransform(transformer.getClass())) { - return true; - } + task -> { + for (Transformer transformer : transformers) { + if (!isCacheableTransform(transformer.getClass())) { + return true; } - for (Relocator relocator : relocators) { - if (!isCacheableRelocator(relocator.getClass())) { - return true; - } + } + for (Relocator relocator : relocators) { + if (!isCacheableRelocator(relocator.getClass())) { + return true; } - return false; } + return false; }); } diff --git a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy index fa120bb04..4c26a3d7d 100644 --- a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy +++ b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy @@ -139,16 +139,15 @@ class ShadowPluginSpec extends PluginSpecification { buildFile << """ |task shadow(type: ${ShadowJar.name}) { - | destinationDir = buildDir - | baseName = 'shadow' + | destinationDirectory = buildDir + | archiveBaseName = 'shadow' | from('${artifact.path}') | from('${project.path}') |} """.stripMargin() when: - runner.arguments << 'shadow' - ExecutionResult result = runner.run() + BuildResult result = run('shadow') then: assert result.output =~ /\s*IGNORING Weird-File\.StrangeFormat from test-project-1\.0-SNAPSHOT\.jar, size is different \([0-9]{4} vs [0-9]{2}\)\s+--> origin JAR was Weird-File.StrangeFormat/ From 2fcc9fef9ae76f25f989bef3a19188ce0db8799b Mon Sep 17 00:00:00 2001 From: Jan Diederich Date: Sat, 7 Sep 2024 20:07:02 +0200 Subject: [PATCH 14/18] Added tests for StandardFilesMergeTransformer --- .../shadow/tasks/ShadowCopyAction.groovy | 6 ++-- .../plugins/shadow/tasks/ShadowJar.java | 22 +++++------- .../plugins/shadow/ShadowPluginSpec.groovy | 34 ++++++++++++++++-- .../resources/test-artifact-1.0-SNAPSHOT.jar | Bin 3725 -> 3906 bytes .../resources/test-project-1.0-SNAPSHOT.jar | Bin 4541 -> 4731 bytes 5 files changed, 43 insertions(+), 19 deletions(-) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy index b564efbc1..8a42e7ddf 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy @@ -8,6 +8,7 @@ import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator import com.github.jengelman.gradle.plugins.shadow.transformers.StandardFilesMergeTransformer import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext +import groovy.util.logging.Slf4j import org.apache.commons.io.FilenameUtils import org.apache.commons.io.IOUtils import org.apache.tools.zip.UnixStat @@ -39,17 +40,14 @@ import org.objectweb.asm.ClassReader import org.objectweb.asm.ClassVisitor import org.objectweb.asm.ClassWriter import org.objectweb.asm.commons.ClassRemapper -import org.slf4j.Logger -import org.slf4j.LoggerFactory import javax.annotation.Nullable import java.util.zip.ZipException +@Slf4j class ShadowCopyAction implements CopyAction { static final long CONSTANT_TIME_FOR_ZIP_ENTRIES = (new GregorianCalendar(1980, 1, 1, 0, 0, 0)).getTimeInMillis() - final static Logger log = LoggerFactory.getLogger(ShadowCopyAction.class); - private final File zipFile private final ZipCompressor compressor private final DocumentationRegistry documentationRegistry diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java index 6065845f0..70fd30b20 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java @@ -1,14 +1,5 @@ package com.github.jengelman.gradle.plugins.shadow.tasks; -import java.io.File; -import java.lang.reflect.InvocationTargetException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.Callable; - -import javax.annotation.Nonnull; - import com.github.jengelman.gradle.plugins.shadow.ShadowStats; import com.github.jengelman.gradle.plugins.shadow.internal.DefaultDependencyFilter; import com.github.jengelman.gradle.plugins.shadow.internal.DependencyFilter; @@ -27,14 +18,12 @@ import com.github.jengelman.gradle.plugins.shadow.transformers.StandardFilesMergeTransformer; import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer; import org.gradle.api.Action; -import org.gradle.api.Task; import org.gradle.api.file.ConfigurableFileCollection; import org.gradle.api.file.DuplicatesStrategy; import org.gradle.api.file.FileCollection; import org.gradle.api.internal.DocumentationRegistry; import org.gradle.api.internal.file.FileResolver; import org.gradle.api.internal.file.copy.CopyAction; -import org.gradle.api.specs.Spec; import org.gradle.api.tasks.CacheableTask; import org.gradle.api.tasks.Classpath; import org.gradle.api.tasks.Input; @@ -47,9 +36,16 @@ import org.gradle.api.tasks.SourceSet; import org.gradle.api.tasks.SourceSetContainer; import org.gradle.api.tasks.TaskAction; -import org.gradle.api.tasks.*; import org.gradle.api.tasks.bundling.Jar; import org.gradle.api.tasks.util.PatternSet; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.lang.reflect.InvocationTargetException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Callable; @CacheableTask public class ShadowJar extends Jar implements ShadowSpec { @@ -141,7 +137,7 @@ public InheritManifest getManifest() { } @Override - @Nonnull + @NotNull protected CopyAction createCopyAction() { DocumentationRegistry documentationRegistry = getServices().get(DocumentationRegistry.class); final UnusedTracker unusedTracker = minimizeJar ? UnusedTracker.forProject(getApiJars(), diff --git a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy index 4c26a3d7d..d6ea4d2e0 100644 --- a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy +++ b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy @@ -150,7 +150,38 @@ class ShadowPluginSpec extends PluginSpecification { BuildResult result = run('shadow') then: - assert result.output =~ /\s*IGNORING Weird-File\.StrangeFormat from test-project-1\.0-SNAPSHOT\.jar, size is different \([0-9]{4} vs [0-9]{2}\)\s+--> origin JAR was Weird-File.StrangeFormat/ + assert result.output =~ /\s*IGNORING Weird-File\.StrangeFormat from test-project-1\.0-SNAPSHOT\.jar,/ + / size is different \([0-9]{4} vs [0-9]{2}\)\s+--> origin JAR was Weird-File.StrangeFormat/ + + /* Shouldn't appear, because the default StandardFileTransformer should've merged it, + instead of just dropping all following licenses. */ + assert !result.output.contains('license.txt') + } + + def 'Tests the removal of the default transformer'() { + given: + URL artifact = this.class.classLoader.getResource('test-artifact-1.0-SNAPSHOT.jar') + URL project = this.class.classLoader.getResource('test-project-1.0-SNAPSHOT.jar') + + buildFile << """ + |task shadow(type: ${ShadowJar.name}) { + | destinationDirectory = buildDir + | archiveBaseName = 'shadow' + | removeDefaultTransformers() + | from('${artifact.path}') + | from('${project.path}') + |} + """.stripMargin() + + when: + BuildResult result = run('shadow') + + then: + assert result.output =~ /\s*IGNORING Weird-File\.StrangeFormat from test-project-1\.0-SNAPSHOT\.jar,/ + / size is different \([0-9]{4} vs [0-9]{2}\)\s+--> origin JAR was Weird-File.StrangeFormat/ + /\s+IGNORING test\.json from test-project-1\.0-SNAPSHOT.jar, size is different/ + // Without the StandardFileTransformer there should be a warning about multiple license files with the same name. + assert result.output.contains('license.txt') } def 'include project sources'() { @@ -214,7 +245,6 @@ class ShadowPluginSpec extends PluginSpecification { repositories { maven { url "${repo.uri}" } } dependencies { implementation project(':client') } - """.stripIndent() File serverOutput = getFile('server/build/libs/server-all.jar') diff --git a/src/test/resources/test-artifact-1.0-SNAPSHOT.jar b/src/test/resources/test-artifact-1.0-SNAPSHOT.jar index dbe60ba230a033e51668b8fa924b9f141bf62d1f..8bc765711892594930293a8054285afbe67d7009 100644 GIT binary patch delta 403 zcmeB`JtQ~bkhcf}0|N&`<6QN~{~CM~WPv;lAO?y0x`sIFdiuHP=VT_Q<`t*vl~j~` zKI!9kR@*Ps;0f2s6FOlUCp|&~nz}VL9cEU=Fien|EXjI!vNik5&9_+N87Hf7nIG7#Q-4()A}da>`8h;NoN2&oViJ%be*2 z%jAh%woKxzKuUP>JVuVmuer3C{9s&x$yVGNOx-XpAE!@dl3sF7VsY{08@yVK_a-y* Z`7s7h4&u}06J!u%h+$=5xWEbW1pq)yX|wk*DE{W&_ouo&D&YDS%CCpb;iS!&3HsMzhv@cocx0E_2i{IB9q-%qBlR{$Yz{; zg;PR|g#iWx7~VSW19I|<()A}_;FOu{&dbO2fMIetuQ?MZB)6`Y?ICTw3ybwxWXV4C!glxWBSiB`8Jm|lMd_Td)x+-J-D@)DnLxT$yz*GOqW1R zC+3RWoXMMjGKTCxZouUEJZelb9Fvdpm`#4b$1!;(pB!WV^1_lm>>2uX1yUhhe)quPRAO?y0x`sIFdiuHP=VT_Q<`t*vl~j~$ zKFYL1#NmLtv8CyHi_n7|+^eP5P3Sq3)HH=TJ+l*iVBw}(tzs%&He0)rjjFUO}t(n{y zC(CkZPcG!wVCrU^{FmQsvXy`q(_IkLYVraBzR6el)tI(30lDszrFj*Y;#nr!^2$v1 z;ZtDhWtp7Er!sk+fEv>?j>*>q%qENSN=(k?l({FL>>k z?r~1u#V0knk57w9gbT=2pUfku#`t!!v7jGg@#H2!T|OZOA%;>`28ItjAO`~gP1A3Y delta 285 zcmeyZvR8S+(aEWtVv{RbWF}{F?%X_^C75M$qoCMkBd-076IsM28#8X%e2{4w>*^ak2rI$Yeu7Ev6h0Q)+U#puptWECQ2%@pCaPVw}8^MFA+J#q^sMD62dGBCO2r}@q KGBC^l8v+1V#ZHF+ From 07b97479661944e5e445d328f6262492ba0b8627 Mon Sep 17 00:00:00 2001 From: Jan Diederich Date: Sat, 7 Sep 2024 21:55:50 +0200 Subject: [PATCH 15/18] Adding option allowModuleInfos() to make it optional to include module-info.class files easily --- .../plugins/shadow/ShadowJavaPlugin.groovy | 17 +++ .../shadow/internal/RelocationUtil.groovy | 3 +- .../shadow/tasks/ShadowCopyAction.groovy | 99 ++++++++++++------ .../plugins/shadow/tasks/ShadowJar.java | 22 +++- .../plugins/shadow/ShadowPluginSpec.groovy | 30 +++++- .../resources/test-artifact-1.0-SNAPSHOT.jar | Bin 3906 -> 4848 bytes .../resources/test-project-1.0-SNAPSHOT.jar | Bin 4731 -> 5711 bytes 7 files changed, 134 insertions(+), 37 deletions(-) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowJavaPlugin.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowJavaPlugin.groovy index 7a904a218..244f7a87d 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowJavaPlugin.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowJavaPlugin.groovy @@ -24,6 +24,9 @@ class ShadowJavaPlugin implements Plugin { public static final String SHADOW_GROUP = 'Shadow' public static final String SHADOW_RUNTIME_ELEMENTS_CONFIGURATION_NAME = 'shadowRuntimeElements' + public static final String MODULE_INFO_CLASS = 'module-info.class' + + private final ProjectConfigurationActionContainer configurationActionContainer private final SoftwareComponentFactory softwareComponentFactory @Inject @@ -101,6 +104,20 @@ class ShadowJavaPlugin implements Plugin { project.configurations.runtime, ] shadow.exclude('META-INF/INDEX.LIST', 'META-INF/*.SF', 'META-INF/*.DSA', 'META-INF/*.RSA', 'module-info.class') + shadow.configurations = [project.configurations.findByName('runtimeClasspath') ? + project.configurations.runtimeClasspath : project.configurations.runtime] + /* + Remove excludes like this: + shadowJar { + ... + allowModuleInfos() + } + */ + def excludes = ['META-INF/INDEX.LIST', 'META-INF/*.SF', 'META-INF/*.DSA', 'META-INF/*.RSA'] + if (!shadow.isAllowModuleInfos()) { + excludes.add(MODULE_INFO_CLASS) + } + shadow.exclude(excludes) } project.artifacts.add(shadowConfiguration.name, taskProvider) return taskProvider diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/RelocationUtil.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/RelocationUtil.groovy index f94ab89ed..6054ddfab 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/RelocationUtil.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/internal/RelocationUtil.groovy @@ -1,5 +1,6 @@ package com.github.jengelman.gradle.plugins.shadow.internal +import com.github.jengelman.gradle.plugins.shadow.ShadowJavaPlugin import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar import java.util.jar.JarFile @@ -11,7 +12,7 @@ class RelocationUtil { configuration.files.each { jar -> JarFile jf = new JarFile(jar) jf.entries().each { entry -> - if (entry.name.endsWith(".class") && entry.name != "module-info.class") { + if (entry.name.endsWith(".class") && entry.name != ShadowJavaPlugin.MODULE_INFO_CLASS) { packages << entry.name[0..entry.name.lastIndexOf('/') - 1].replaceAll('/', '.') } } diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy index 8a42e7ddf..da200bb61 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowCopyAction.groovy @@ -1,5 +1,6 @@ package com.github.jengelman.gradle.plugins.shadow.tasks +import com.github.jengelman.gradle.plugins.shadow.ShadowJavaPlugin import com.github.jengelman.gradle.plugins.shadow.ShadowStats import com.github.jengelman.gradle.plugins.shadow.impl.RelocatorRemapper import com.github.jengelman.gradle.plugins.shadow.internal.UnusedTracker @@ -41,6 +42,7 @@ import org.objectweb.asm.ClassVisitor import org.objectweb.asm.ClassWriter import org.objectweb.asm.commons.ClassRemapper +import javax.annotation.Nonnull import javax.annotation.Nullable import java.util.zip.ZipException @@ -59,11 +61,13 @@ class ShadowCopyAction implements CopyAction { private final boolean preserveFileTimestamps private final boolean minimizeJar private final UnusedTracker unusedTracker + private final boolean allowModuleInfos ShadowCopyAction(File zipFile, ZipCompressor compressor, DocumentationRegistry documentationRegistry, String encoding, List transformers, List relocators, PatternSet patternSet, ShadowStats stats, - boolean preserveFileTimestamps, boolean minimizeJar, UnusedTracker unusedTracker) { + boolean preserveFileTimestamps, boolean minimizeJar, UnusedTracker unusedTracker, + boolean allowModuleInfos) { this.zipFile = zipFile this.compressor = compressor @@ -76,6 +80,7 @@ class ShadowCopyAction implements CopyAction { this.preserveFileTimestamps = preserveFileTimestamps this.minimizeJar = minimizeJar this.unusedTracker = unusedTracker + this.allowModuleInfos = allowModuleInfos } @Override @@ -201,7 +206,17 @@ class ShadowCopyAction implements CopyAction { private final Set unused private final ShadowStats stats - private Map visitedFiles = new HashMap<>() + private class VisitedFileInfo { + long size + RelativePath originJar + + VisitedFileInfo(long size, @Nonnull RelativePath originJar) { + this.size = size + this.originJar = originJar + } + } + + private Map visitedFiles = new HashMap<>() StreamAction(ZipOutputStream zipOutStr, String encoding, List transformers, List relocators, PatternSet patternSet, Set unused, @@ -235,7 +250,7 @@ class ShadowCopyAction implements CopyAction { originJar = new RelativePath(false) } - visitedFiles.put(path.toString(), [size: size, originJar: originJar]) + visitedFiles.put(path.toString(), new VisitedFileInfo(size, originJar)) return true } @@ -312,37 +327,7 @@ class ShadowCopyAction implements CopyAction { if (archiveFile.classFile || !isTransformable(archiveFile)) { String path = archiveFilePath.toString() - if (path.endsWith("module-info.class")) { - log.warn("module-info collision") - - def moduleFileName = "module-info" - def moduleFileSuffix = ".class" - File disassembleModFile = File.createTempFile(moduleFileName, moduleFileSuffix) - - try (InputStream is = archive.getInputStream(archiveFilePath.entry)) { - try (OutputStream os = new FileOutputStream(disassembleModFile)) { - IOUtils.copyLarge(is, os) - } - } - - ProcessBuilder processBuilder = new ProcessBuilder("javap", disassembleModFile.absolutePath) - processBuilder.redirectErrorStream(true) - Process process = processBuilder.start() - InputStream inputStream = process.getInputStream() - BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)) - String line - - while ((line = reader.readLine()) != null) { - log.warn(line) - } - - int exitCode = process.waitFor() - if (exitCode != 0) { - log.warn("Process exited with code " + exitCode) - } - - log.warn("module-info collision end") - } + listModuleInfoOnDemand(path, archive, archiveFilePath) if (recordVisit(path, archiveFileSize, archiveFilePath) && !isUnused(archiveFilePath.entry.name)) { if (!remapper.hasRelocators() || !archiveFile.classFile) { @@ -382,6 +367,52 @@ class ShadowCopyAction implements CopyAction { } } + /** + Information about the 'module-info.class' if it isn't excluded. Including can be done with + allowModuleInfos(), like this: + 

+         shadowJar {
+            ...
+            allowModuleInfos()
+         }
+
+ Based on the discussion in issue 710: GitHub Issue #710. + */ + private void listModuleInfoOnDemand(String path, ZipFile archive, RelativeArchivePath archiveFilePath) { + if (path.endsWith(ShadowJavaPlugin.MODULE_INFO_CLASS) && allowModuleInfos) { + log.warn("======== Warning: {}/{} contains module-info - Listing content ========", + RelativePath.parse(true, archive.name).lastName, path) + + def moduleFileName = "module-info" + def moduleFileSuffix = ".class" + File disassembleModFile = File.createTempFile(moduleFileName, moduleFileSuffix) + + try (InputStream is = archive.getInputStream(archiveFilePath.entry)) { + try (OutputStream os = new FileOutputStream(disassembleModFile)) { + IOUtils.copyLarge(is, os) + } + } + + ProcessBuilder processBuilder = new ProcessBuilder("javap", disassembleModFile.absolutePath) + processBuilder.redirectErrorStream(true) + Process process = processBuilder.start() + InputStream inputStream = process.getInputStream() + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)) + + String line + while ((line = reader.readLine()) != null) { + log.warn(line) + } + + int exitCode = process.waitFor() + if (exitCode != 0) { + log.warn("Process exited with code " + exitCode) + } + + log.warn("======== module-info content listing end ========") + } + } + private void addParentDirectories(RelativeArchivePath file) { if (file) { addParentDirectories(file.parent) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java index 70fd30b20..31620d3d3 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/tasks/ShadowJar.java @@ -1,5 +1,6 @@ package com.github.jengelman.gradle.plugins.shadow.tasks; +import com.github.jengelman.gradle.plugins.shadow.ShadowJavaPlugin; import com.github.jengelman.gradle.plugins.shadow.ShadowStats; import com.github.jengelman.gradle.plugins.shadow.internal.DefaultDependencyFilter; import com.github.jengelman.gradle.plugins.shadow.internal.DependencyFilter; @@ -72,6 +73,8 @@ public FileCollection call() { } }); + private boolean isAllowModuleInfos; + public ShadowJar() { super(); setDuplicatesStrategy( @@ -144,7 +147,7 @@ protected CopyAction createCopyAction() { getSourceSetsClassesDirs().getFiles(), getToMinimize()) : null; return new ShadowCopyAction(getArchiveFile().get().getAsFile(), getInternalCompressor(), documentationRegistry, this.getMetadataCharset(), transformers, relocators, getRootPatternSet(), shadowStats, - isPreserveFileTimestamps(), minimizeJar, unusedTracker); + isPreserveFileTimestamps(), minimizeJar, unusedTracker, isAllowModuleInfos); } @Classpath @@ -269,6 +272,23 @@ public ShadowJar removeDefaultTransformers() { return this; } + /** + * Allows module-info.class's to be included in the final jar, and informs about the contents + * of the module-info.class files it finds. + * + * @return this + */ + public ShadowJar allowModuleInfos() { + this.isAllowModuleInfos = true; + getExcludes().remove(ShadowJavaPlugin.MODULE_INFO_CLASS); + return this; + } + + @Internal + public boolean isAllowModuleInfos() { + return isAllowModuleInfos; + } + private boolean isCacheableTransform(Class clazz) { return clazz.isAnnotationPresent(CacheableTransformer.class); } diff --git a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy index d6ea4d2e0..017e2c40f 100644 --- a/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy +++ b/src/test/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowPluginSpec.groovy @@ -156,6 +156,8 @@ class ShadowPluginSpec extends PluginSpecification { /* Shouldn't appear, because the default StandardFileTransformer should've merged it, instead of just dropping all following licenses. */ assert !result.output.contains('license.txt') + // No module listing should appear. module-info.class is excluded by default. + assert !result.output.contains('open module org.example') } def 'Tests the removal of the default transformer'() { @@ -167,7 +169,7 @@ class ShadowPluginSpec extends PluginSpecification { |task shadow(type: ${ShadowJar.name}) { | destinationDirectory = buildDir | archiveBaseName = 'shadow' - | removeDefaultTransformers() + | removeDefaultTransformers() | from('${artifact.path}') | from('${project.path}') |} @@ -182,6 +184,32 @@ class ShadowPluginSpec extends PluginSpecification { /\s+IGNORING test\.json from test-project-1\.0-SNAPSHOT.jar, size is different/ // Without the StandardFileTransformer there should be a warning about multiple license files with the same name. assert result.output.contains('license.txt') + // No module listing should appear. module-info.class is excluded by default. + assert !result.output.contains('open module org.example') + } + + def 'Tests the info about the module-info.class, if removed from standard excludes'() { + given: + URL artifact = this.class.classLoader.getResource('test-artifact-1.0-SNAPSHOT.jar') + URL project = this.class.classLoader.getResource('test-project-1.0-SNAPSHOT.jar') + + buildFile << """ + |task shadow(type: ${ShadowJar.name}) { + | destinationDirectory = buildDir + | archiveBaseName = 'shadow' + | allowModuleInfos() + | from('${artifact.path}') + | from('${project.path}') + |} + """.stripMargin() + + when: + BuildResult result = run('shadow') + + then: + assert result.output.contains('module-info.class') + // Because allowModuleInfos() is used, the module listing should appear. + assert result.output.contains('open module org.example') } def 'include project sources'() { diff --git a/src/test/resources/test-artifact-1.0-SNAPSHOT.jar b/src/test/resources/test-artifact-1.0-SNAPSHOT.jar index 8bc765711892594930293a8054285afbe67d7009..d12faa8977c31d8da5da81ec4d0d1242d1c67f9b 100644 GIT binary patch delta 1081 zcmX>k_d#{TLY8_F1_lNW2IeY_$Yxf7ukAqIb|4l6;@td{(wtP?%)GRGz2uz4;$rKs z`Pc2(*w~`k;>y_6+}PgT`7`CxoRGU`_H3zfV_Wy&&!t6ALbjio6XM3ksG_-d&5=zm zZfs`p8zvYqTL+}q1@4N=a!tx(^k|qou~B%5o4)2oi4~61?_P*G?{NNtM@u97q9FNI zn;IJ%Co;0lkeG2$g_~J>!nDMM*#1wa4X>pobw#egYY_{alxe?rQ+df4k6`Xe!SaGZQ(!mAq7IdU? ztBR+tF_`jviNu{w#!-#d)Ew za-k(>wbT6%aSlE!*Z#73QKWme)+_CiwWZmu|BJ*j6N4gh@0`2sZ*|=+^>lFx+w#R~ zyY$a3xtTQk81rK%SmOE2;I7I z=$lDR#wnifOslv2=sJ|&=$zqH`6SIN=S{-&2@9uLyuQEr9)~-g9voAYTg zIl{QYP@iF1%LB9)Ve1r5)Rc)~J6=y=OTSRlof$v@w-3#DL~i;;@td{(wtP?%)GRGz2uz4;$rKs z`Pc2(*w~`k;>y_6+}PgT`7`CxoRGU`_H3zfV_Wy&&!t6ALbjio6XM3ksG_-d&5=zm zZfs`p8zvYqTL+}q1@4N=a!tx(^k|qou~B%5o4)2oi50HX?_P*G?{NNtM@wV-#76cN zLGr6Et!Qj)oXE&FLt@536>etj3DXi2k^>SLP8KtS7BXh$u~{%M1b8#EfSk?%cKYUg z&Ne1C2z&BGPLcZ5ip1Q4oKz!ym^z3TC0EuZ%IW{3{>+Z@Y6Rt0ij%YJf-yZKa@q_2mV=1d5IF+YXN}p2Lv^bz(|K!73 z&4sEq{-*OnUVNQx)fI6$z$oCDm2+idx1;p2x#9sir+x?CG|3 zYI9qZ^N(rsTkhE`_4drT5}*nwa)CghVM}8oCzj;M1&%@n21W)62G+mn2R}0221YF^ z2=HcPf>;4b9LTaP3@QxD5CMUPzmC#eko16TY^!PJ!H<*o@XAmADOQj^Pp%yS^7ay>rld4bj=Y+ufUmKrgff!BA~lD7&& z1jKw$2wp-rA4_T{)=D|%$sf5T_$0ulgMxfZV3hD5PGKeyq1!l(_ HevlFXK(lHN delta 139 zcmX@F^IK&@D(B>67O~ByTvbeymxzjNZsn_IVG?1U?8qf1!@>Xq0t{~*6FC_e@{7{- z1H2g_3Z!@-(jduZ&dE=OEt%GFPF4`HW4gyVIYGpQQDkzwi1g$GBD|C5iK;QZ<(_;@ Z)R(E4XR@f64xbQ%5JM>|1H%WfApky6AVUBE From 9e211d274ae5d5223f11f4c8be125e033c0ca371 Mon Sep 17 00:00:00 2001 From: Jan Diederich Date: Thu, 19 Sep 2024 00:21:16 +0200 Subject: [PATCH 16/18] Fixed rebase errors with module-info exclusion and moved dependencies --- .../jengelman/gradle/plugins/shadow/ShadowJavaPlugin.groovy | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowJavaPlugin.groovy b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowJavaPlugin.groovy index 244f7a87d..4f7c4404b 100644 --- a/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowJavaPlugin.groovy +++ b/src/main/groovy/com/github/jengelman/gradle/plugins/shadow/ShadowJavaPlugin.groovy @@ -26,7 +26,6 @@ class ShadowJavaPlugin implements Plugin { public static final String MODULE_INFO_CLASS = 'module-info.class' - private final ProjectConfigurationActionContainer configurationActionContainer private final SoftwareComponentFactory softwareComponentFactory @Inject @@ -103,9 +102,6 @@ class ShadowJavaPlugin implements Plugin { project.configurations.findByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME) ?: project.configurations.runtime, ] - shadow.exclude('META-INF/INDEX.LIST', 'META-INF/*.SF', 'META-INF/*.DSA', 'META-INF/*.RSA', 'module-info.class') - shadow.configurations = [project.configurations.findByName('runtimeClasspath') ? - project.configurations.runtimeClasspath : project.configurations.runtime] /* Remove excludes like this: shadowJar { From f5a57ae7b2b4e3e133a31b412d6e5159603a8be5 Mon Sep 17 00:00:00 2001 From: Jan Diederich Date: Fri, 20 Sep 2024 00:29:13 +0200 Subject: [PATCH 17/18] Removed package lock-file from .gitignore --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 6927bd7ee..646e97241 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,7 @@ classes/ node_modules/ yarn-error.log src/docs/.vuepress/dist/ -bin/ .DS_Store jd-gui.cfg +bin/ .vscode/ -package-lock.json From dba0157b69e6f99e201b34f3389a426044527723 Mon Sep 17 00:00:00 2001 From: Jan Diederich Date: Fri, 20 Sep 2024 00:58:08 +0200 Subject: [PATCH 18/18] Fixed rebase conflict in build.gradle.kts --- build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/build.gradle.kts b/build.gradle.kts index fc45934d0..ff179cd63 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -28,6 +28,7 @@ dependencies { implementation("org.codehaus.plexus:plexus-xml:4.0.4") implementation("org.apache.logging.log4j:log4j-core:2.24.0") implementation("org.vafer:jdependency:2.11") + implementation("com.google.code.gson:gson:2.11.0") testImplementation("org.spockframework:spock-core:2.3-groovy-3.0") { exclude(group = "org.codehaus.groovy")