Merge branch 'main' into 213-improve-sourcecodesecuritycheckstep

.github/scripts/assignments_test_docker.py

@@ -21,51 +21,51 @@ def get_directories(basedir):
 
 pipeline_failed = False
 for category_dir in get_directories(home_dir):
-    #for assignment_dir in get_directories(category_dir):
-    assignment_dir = random.choice(get_directories(category_dir));
-    # Remove the contents of the output directory.
-    os.system(f'rm -r {output_dir}')
-
-    # Remove the contents of the test directory.
-    os.system(f'rm -r {test_dir}/*')
-
-    # Write environment file
-    with open(f'{test_dir}/.env', 'w') as envfile:
-        envfile.write('TASK_MODE=FULL_WITH_HINTS')
-
-    # Copy the assignment to the test folder.
-    os.chdir(assignment_dir)
-    os.system(f'cp ./config/Configuration.java {test_dir}/test.txt')
-    os.system(f'cp ./solution/*.java {test_dir}/solution.txt')
-    os.system(f'cp ./src/main/java/delft/*.java {test_dir}/library.txt')
-    # Copy resources
-    os.system('find . -type f | ' +
-              'grep -i -v "^\./src/" | grep -i -v "\./config/Configuration.java" | ' +
-              'grep -i -v "^\./pom.xml$" | grep -i -v "^\./solution/" | grep -i -v "^\./README.md$" | ' +
-              'xargs -i cp --parents {} ' + f'{test_dir}/')
-
-    # Switch to Docker directory
-    os.chdir(docker_dir)
-
-    # Run `andy` on the current assignment.
-    output = os.popen('make github-ci.test').read()
-
-    re_score = re.search('Final grade: [0-9]+', output)
-    score = int(re_score.group().split()[2]) if re_score else -1
-    re_andy_version = re.search('Andy v.+', output)
-    andy_version = re_andy_version.group() if re_andy_version else "Unknown Andy version"
-
-    # Print the score for the assignment.
-    print(f'{andy_version} | {assignment_dir.split("/")[-2]}/{assignment_dir.split("/")[-1]}: {score}/100')
-
-    # Update the `pipeline_failed` variable.
-    if score != 100:
-        print(output)
-        pipeline_failed = True
-
-    if expected_andy_version not in andy_version:
-        print(f'Error: Unexpected Andy version {andy_version}, expected {expected_andy_version}')
-        pipeline_failed = True
+    for assignment_dir in random.choices(get_directories(category_dir), k=4):
+        
+        # Remove the contents of the output directory.
+        os.system(f'rm -r {output_dir}')
+
+        # Remove the contents of the test directory.
+        os.system(f'rm -r {test_dir}/*')
+
+        # Write environment file
+        with open(f'{test_dir}/.env', 'w') as envfile:
+            envfile.write('TASK_MODE=FULL_WITH_HINTS')
+
+        # Copy the assignment to the test folder.
+        os.chdir(assignment_dir)
+        os.system(f'cp ./config/Configuration.java {test_dir}/test.txt')
+        os.system(f'cp ./solution/*.java {test_dir}/solution.txt')
+        os.system(f'cp ./src/main/java/delft/*.java {test_dir}/library.txt')
+        # Copy resources
+        os.system('find . -type f | ' +
+                  'grep -i -v "^\./src/" | grep -i -v "\./config/Configuration.java" | ' +
+                  'grep -i -v "^\./pom.xml$" | grep -i -v "^\./solution/" | grep -i -v "^\./README.md$" | ' +
+                  'xargs -i cp --parents {} ' + f'{test_dir}/')
+
+        # Switch to Docker directory
+        os.chdir(docker_dir)
+
+        # Run `andy` on the current assignment.
+        output = os.popen('make github-ci.test').read()
+
+        re_score = re.search('Final grade: [0-9]+', output)
+        score = int(re_score.group().split()[2]) if re_score else -1
+        re_andy_version = re.search('Andy v.+', output)
+        andy_version = re_andy_version.group() if re_andy_version else "Unknown Andy version"
+
+        # Print the score for the assignment.
+        print(f'{andy_version} | {assignment_dir.split("/")[-2]}/{assignment_dir.split("/")[-1]}: {score}/100')
+
+        # Update the `pipeline_failed` variable.
+        if score != 100:
+            print(output)
+            pipeline_failed = True
+
+        if expected_andy_version not in andy_version:
+            print(f'Error: Unexpected Andy version {andy_version}, expected {expected_andy_version}')
+            pipeline_failed = True
 
 if pipeline_failed:
     sys.exit('Some assignments do not have 100/100.')

.github/workflows/tests.yml

@@ -35,6 +35,7 @@ jobs:
         job:
           - os: ubuntu-22.04
           - os: macos-12
+          - os: windows-latest
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
 
     steps:
@@ -66,6 +67,7 @@ jobs:
         job:
           - os: ubuntu-22.04
           - os: macos-12
+          - os: windows-latest
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
 
     steps:
@@ -97,6 +99,7 @@ jobs:
         job:
           - os: ubuntu-22.04
           - os: macos-12
+          - os: windows-latest
     if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
 
     steps:

README.md

@@ -12,6 +12,7 @@
   * [Code coverage](#code-coverage)
   * [Mutation coverage](#mutation-coverage)
   * [Meta tests](#meta-tests)
+  * [Penalty meta tests](#penalty-meta-tests)
   * [Code checks](#code-checks)
   * [Penalty code checks](#penalty-code-checks)
   * [Success message](#success-message)
@@ -285,6 +286,39 @@ Meta test: returns wrong elements (weight: 1) FAILED
 Meta test: compares only elements in the same index (weight: 3) FAILED
 Meta test: background service does not work (weight: 2) PASSED
 ```
+#### Penalty meta tests
+
+Penalty meta tests work in a similar way to regular meta tests. However, they are not considered a grading component. Instead, if a penalty meta test passes, this has no effect on the final score. However, if it fails, its weight is subtracted from the final grade.
+
+Penalty meta tests must have a positive weight, but there is no upper limit on their total weight. For example, if there are two penalty meta tests where one of them has a weight of 10 and the other one 100 (see the example below), and a student fails both of them, the total applied penalty would be 110. In case this makes the final grade negative, the grade is reported as 0 instead.
+
+Penalty meta tests can be defined as follows:
+
+```java
+@Override
+public List<MetaTest> penaltyMetaTests() {
+        return List.of(
+            MetaTest.insertAt(2, "returns empty list when list1 is null", 29,
+                "if (list1 == null) return result;"
+            ),
+            MetaTest.withStringReplacement("returns wrong elements",
+                "if (hashSet.contains(e))",
+                "if (!hashSet.contains(e))"
+            ),
+            MetaTest.withLineReplacement(3, "compares only elements in the same index", 29, 41,
+                    """
+                    for (int i = 0; i < list1.size() && i < list2.size(); i++) {
+                        if (list1.get(i).equals(list2.get(i))) {
+                            result.add(list1.get(i));
+                        }
+                    }
+                    """
+            )
+        );
+}
+```
+
+If this method is not overridden, penalty meta tests are disabled.
 
 #### Code checks
 

andy/src/main/java/nl/tudelft/cse1110/andy/config/RunConfiguration.java

@@ -45,6 +45,10 @@ public List<MetaTest> metaTests() {
         return Collections.emptyList();
     }
 
+    public List<MetaTest> penaltyMetaTests() {
+        return Collections.emptyList();
+    }
+
     public List<String> listOfMutants() {
         return DEFAULTS;
     }

andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java

@@ -16,13 +16,17 @@ public class SecureExamRunConfiguration extends RunConfiguration {
     private final List<String> listOfMutants;
     private final int numberOfMutationsToConsider;
     private final ExternalProcess externalProcess;
+    private final boolean skipJacoco;
+    private final boolean skipPitest;
 
     public SecureExamRunConfiguration(RunConfiguration runConfigurationToClone) {
         this.classesUnderTest = runConfigurationToClone.classesUnderTest();
         this.listOfMutants = runConfigurationToClone.listOfMutants();
         this.numberOfMutationsToConsider = runConfigurationToClone.numberOfMutationsToConsider();
         this.externalProcess = runConfigurationToClone.externalProcess();
         this.successMessage = runConfigurationToClone.successMessage();
+        this.skipJacoco = runConfigurationToClone.skipJacoco();
+        this.skipPitest = runConfigurationToClone.skipPitest();
     }
 
     public Mode mode() {
@@ -60,5 +64,13 @@ public String successMessage() {
         return successMessage;
     }
 
+    @Override
+    public boolean skipJacoco() {
+        return this.skipJacoco;
+    }
 
+    @Override
+    public boolean skipPitest() {
+        return this.skipPitest;
+    }
 }

andy/src/main/java/nl/tudelft/cse1110/andy/execution/mode/ModeActionSelector.java

@@ -129,6 +129,7 @@ public static List<ExecutionStep> fullMode() {
                 new RunPenaltyCodeChecksStep(),
                 new RunCodeChecksStep(),
                 new KillExternalProcessStep(),
+                new RunPenaltyMetaTestsStep(),
                 new RunMetaTestsStep()
         );
     }

andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java

@@ -0,0 +1,62 @@
+package nl.tudelft.cse1110.andy.execution.step;
+
+import nl.tudelft.cse1110.andy.config.DirectoryConfiguration;
+import nl.tudelft.cse1110.andy.config.MetaTest;
+import nl.tudelft.cse1110.andy.config.RunConfiguration;
+import nl.tudelft.cse1110.andy.execution.Context.Context;
+import nl.tudelft.cse1110.andy.execution.ExecutionStep;
+import nl.tudelft.cse1110.andy.result.MetaTestResult;
+import nl.tudelft.cse1110.andy.result.ResultBuilder;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class RunPenaltyMetaTestsStep implements ExecutionStep {
+
+    @Override
+    public void execute(Context ctx, ResultBuilder result) {
+        DirectoryConfiguration dirCfg = ctx.getDirectoryConfiguration();
+        RunConfiguration runCfg = ctx.getRunConfiguration();
+
+        int score = 0;
+        int totalWeight = 0;
+
+        ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader();
+
+        try {
+
+            List<MetaTest> metaTests = runCfg.penaltyMetaTests();
+            List<MetaTestResult> metaTestResults = new ArrayList<>();
+
+            for (MetaTest metaTest : metaTests) {
+
+                boolean passesTheMetaTest = metaTest.execute(ctx, dirCfg, runCfg);
+
+                if (passesTheMetaTest) {
+                    score += metaTest.getWeight();
+                }
+
+                metaTestResults.add(new MetaTestResult(metaTest.getName(), metaTest.getWeight(), passesTheMetaTest));
+
+                totalWeight += metaTest.getWeight();
+            }
+
+            result.logPenaltyMetaTests(score, totalWeight, metaTestResults);
+        } catch (Exception ex) {
+            result.genericFailure(this, ex);
+        } finally {
+            /* restore the class loader to the one before meta tests */
+            Thread.currentThread().setContextClassLoader(currentClassLoader);
+        }
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        return other instanceof RunPenaltyMetaTestsStep;
+    }
+
+    @Override
+    public int hashCode() {
+        return super.hashCode();
+    }
+}

andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java

@@ -86,15 +86,16 @@ public GradeValues setPenalty(int penalty) {
         return this;
     }
 
-    public static GradeValues fromResults(CoverageResult coverageResults, CodeChecksResult codeCheckResults, MutationTestingResult mutationResults, MetaTestsResult metaTestResults, CodeChecksResult penaltyCodeCheckResults) {
+    public static GradeValues fromResults(CoverageResult coverageResults, CodeChecksResult codeCheckResults, MutationTestingResult mutationResults, MetaTestsResult metaTestResults, MetaTestsResult penaltyMetaTestResults, CodeChecksResult penaltyCodeCheckResults) {
         GradeValues grades = new GradeValues();
         grades.setBranchGrade(coverageResults.getCoveredBranches(), coverageResults.getTotalNumberOfBranches());
         grades.setCheckGrade(codeCheckResults.getNumberOfPassedChecks(), codeCheckResults.getTotalNumberOfChecks());
         grades.setMutationGrade(mutationResults.getKilledMutants(), mutationResults.getTotalNumberOfMutants());
         grades.setMetaGrade(metaTestResults.getPassedMetaTests(), metaTestResults.getTotalTests());
 
         // penalty is equal to the sum of the weights of all failed penalty code checks
-        grades.setPenalty(penaltyCodeCheckResults.getCheckResults().stream().mapToInt(check -> check.passed() ? 0 : check.getWeight()).sum());
+        grades.setPenalty(penaltyCodeCheckResults.getCheckResults().stream().mapToInt(check -> check.passed() ? 0 : check.getWeight()).sum()
+                + penaltyMetaTestResults.getMetaTestResults().stream().mapToInt(check -> check.succeeded() ? 0 : check.getWeight()).sum());
 
         return grades;
     }

andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java

@@ -13,21 +13,23 @@ public class Result {
     private final CodeChecksResult penaltyCodeChecks;
     private final CoverageResult coverage;
     private final MetaTestsResult metaTests;
+    private final MetaTestsResult penaltyMetaTests;
     private final int penalty;
     private final int finalGrade;
     private final GenericFailure genericFailure;
     private final double timeInSeconds;
     private final GradeWeight weights;
     private final String successMessage;
 
-    public Result(CompilationResult compilation, UnitTestsResult tests, MutationTestingResult mutationTesting, CodeChecksResult codeChecks, CodeChecksResult penaltyCodeChecks, CoverageResult coverage, MetaTestsResult metaTests, int penalty, int finalGrade, GenericFailure genericFailure, double timeInSeconds, GradeWeight weights, String successMessage) {
+    public Result(CompilationResult compilation, UnitTestsResult tests, MutationTestingResult mutationTesting, CodeChecksResult codeChecks, CodeChecksResult penaltyCodeChecks, CoverageResult coverage, MetaTestsResult metaTests, MetaTestsResult penaltyMetaTests, int penalty, int finalGrade, GenericFailure genericFailure, double timeInSeconds, GradeWeight weights, String successMessage) {
         this.compilation = compilation;
         this.tests = tests;
         this.mutationTesting = mutationTesting;
         this.codeChecks = codeChecks;
         this.penaltyCodeChecks = penaltyCodeChecks;
         this.coverage = coverage;
         this.metaTests = metaTests;
+        this.penaltyMetaTests = penaltyMetaTests;
         this.penalty = penalty;
         this.finalGrade = finalGrade;
         this.genericFailure = genericFailure;
@@ -40,7 +42,7 @@ public Result(CompilationResult compilation, UnitTestsResult tests, MutationTest
     }
 
     public Result(CompilationResult compilation, double timeInSeconds) {
-        this(compilation, UnitTestsResult.empty(), MutationTestingResult.empty(), CodeChecksResult.empty(), CodeChecksResult.empty(), CoverageResult.empty(), MetaTestsResult.empty(), 0, 0, GenericFailure.noFailure(), timeInSeconds, null, null);
+        this(compilation, UnitTestsResult.empty(), MutationTestingResult.empty(), CodeChecksResult.empty(), CodeChecksResult.empty(), CoverageResult.empty(), MetaTestsResult.empty(), MetaTestsResult.empty(), 0, 0, GenericFailure.noFailure(), timeInSeconds, null, null);
     }
 
     public CompilationResult getCompilation() {
@@ -70,6 +72,9 @@ public CoverageResult getCoverage() {
     public MetaTestsResult getMetaTests() {
         return metaTests;
     }
+    public MetaTestsResult getPenaltyMetaTests() {
+        return penaltyMetaTests;
+    }
 
     public int getPenalty() {
         return penalty;

andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java

@@ -49,6 +49,7 @@ public class ResultBuilder {
     private CodeChecksResult penaltyCodeCheckResults = CodeChecksResult.empty();
     private CoverageResult coverageResults = CoverageResult.empty();
     private MetaTestsResult metaTestResults = MetaTestsResult.empty();
+    private MetaTestsResult penaltyMetaTestResults = MetaTestsResult.empty();
 
     public ResultBuilder(Context ctx, GradeCalculator gradeCalculator) {
         this.ctx = ctx;
@@ -244,6 +245,10 @@ public void logMetaTests(int score, int totalTests, List<MetaTestResult> metaTes
         this.metaTestResults.addResults(score, totalTests, metaTestResults);
     }
 
+    public void logPenaltyMetaTests(int score, int totalTests, List<MetaTestResult> metaTestResults) {
+        this.penaltyMetaTestResults.addResults(score, totalTests, metaTestResults);
+    }
+
     /*
      * Generic failures
      */
@@ -271,7 +276,7 @@ public Result build() {
         if(!compilation.successful()) {
             return new Result(compilation, timeInSeconds);
         } else {
-            GradeValues grades = GradeValues.fromResults(coverageResults, codeCheckResults, mutationResults, metaTestResults, penaltyCodeCheckResults);
+            GradeValues grades = GradeValues.fromResults(coverageResults, codeCheckResults, mutationResults, metaTestResults, penaltyMetaTestResults, penaltyCodeCheckResults);
             GradeWeight weights = GradeWeight.fromConfig(ctx.getRunConfiguration().weights());
             String successMessage = ctx.getRunConfiguration().successMessage();
 
@@ -287,6 +292,7 @@ public Result build() {
                     penaltyCodeCheckResults,
                     coverageResults,
                     metaTestResults,
+                    penaltyMetaTestResults,
                     penalty,
                     finalGrade,
                     genericFailureObject,