diff --git a/.github/scripts/assignments_test_docker.py b/.github/scripts/assignments_test_docker.py index 336080d31..5808dbf60 100644 --- a/.github/scripts/assignments_test_docker.py +++ b/.github/scripts/assignments_test_docker.py @@ -21,51 +21,51 @@ def get_directories(basedir): pipeline_failed = False for category_dir in get_directories(home_dir): - #for assignment_dir in get_directories(category_dir): - assignment_dir = random.choice(get_directories(category_dir)); - # Remove the contents of the output directory. - os.system(f'rm -r {output_dir}') - - # Remove the contents of the test directory. - os.system(f'rm -r {test_dir}/*') - - # Write environment file - with open(f'{test_dir}/.env', 'w') as envfile: - envfile.write('TASK_MODE=FULL_WITH_HINTS') - - # Copy the assignment to the test folder. - os.chdir(assignment_dir) - os.system(f'cp ./config/Configuration.java {test_dir}/test.txt') - os.system(f'cp ./solution/*.java {test_dir}/solution.txt') - os.system(f'cp ./src/main/java/delft/*.java {test_dir}/library.txt') - # Copy resources - os.system('find . -type f | ' + - 'grep -i -v "^\./src/" | grep -i -v "\./config/Configuration.java" | ' + - 'grep -i -v "^\./pom.xml$" | grep -i -v "^\./solution/" | grep -i -v "^\./README.md$" | ' + - 'xargs -i cp --parents {} ' + f'{test_dir}/') - - # Switch to Docker directory - os.chdir(docker_dir) - - # Run `andy` on the current assignment. - output = os.popen('make github-ci.test').read() - - re_score = re.search('Final grade: [0-9]+', output) - score = int(re_score.group().split()[2]) if re_score else -1 - re_andy_version = re.search('Andy v.+', output) - andy_version = re_andy_version.group() if re_andy_version else "Unknown Andy version" - - # Print the score for the assignment. - print(f'{andy_version} | {assignment_dir.split("/")[-2]}/{assignment_dir.split("/")[-1]}: {score}/100') - - # Update the `pipeline_failed` variable. - if score != 100: - print(output) - pipeline_failed = True - - if expected_andy_version not in andy_version: - print(f'Error: Unexpected Andy version {andy_version}, expected {expected_andy_version}') - pipeline_failed = True + for assignment_dir in random.choices(get_directories(category_dir), k=4): + + # Remove the contents of the output directory. + os.system(f'rm -r {output_dir}') + + # Remove the contents of the test directory. + os.system(f'rm -r {test_dir}/*') + + # Write environment file + with open(f'{test_dir}/.env', 'w') as envfile: + envfile.write('TASK_MODE=FULL_WITH_HINTS') + + # Copy the assignment to the test folder. + os.chdir(assignment_dir) + os.system(f'cp ./config/Configuration.java {test_dir}/test.txt') + os.system(f'cp ./solution/*.java {test_dir}/solution.txt') + os.system(f'cp ./src/main/java/delft/*.java {test_dir}/library.txt') + # Copy resources + os.system('find . -type f | ' + + 'grep -i -v "^\./src/" | grep -i -v "\./config/Configuration.java" | ' + + 'grep -i -v "^\./pom.xml$" | grep -i -v "^\./solution/" | grep -i -v "^\./README.md$" | ' + + 'xargs -i cp --parents {} ' + f'{test_dir}/') + + # Switch to Docker directory + os.chdir(docker_dir) + + # Run `andy` on the current assignment. + output = os.popen('make github-ci.test').read() + + re_score = re.search('Final grade: [0-9]+', output) + score = int(re_score.group().split()[2]) if re_score else -1 + re_andy_version = re.search('Andy v.+', output) + andy_version = re_andy_version.group() if re_andy_version else "Unknown Andy version" + + # Print the score for the assignment. + print(f'{andy_version} | {assignment_dir.split("/")[-2]}/{assignment_dir.split("/")[-1]}: {score}/100') + + # Update the `pipeline_failed` variable. + if score != 100: + print(output) + pipeline_failed = True + + if expected_andy_version not in andy_version: + print(f'Error: Unexpected Andy version {andy_version}, expected {expected_andy_version}') + pipeline_failed = True if pipeline_failed: sys.exit('Some assignments do not have 100/100.') diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3dd3c836b..bff9f4744 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,6 +35,7 @@ jobs: job: - os: ubuntu-22.04 - os: macos-12 + - os: windows-latest if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name steps: @@ -66,6 +67,7 @@ jobs: job: - os: ubuntu-22.04 - os: macos-12 + - os: windows-latest if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name steps: @@ -97,6 +99,7 @@ jobs: job: - os: ubuntu-22.04 - os: macos-12 + - os: windows-latest if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name steps: diff --git a/README.md b/README.md index ed87b9f5e..cb6f297c5 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ * [Code coverage](#code-coverage) * [Mutation coverage](#mutation-coverage) * [Meta tests](#meta-tests) + * [Penalty meta tests](#penalty-meta-tests) * [Code checks](#code-checks) * [Penalty code checks](#penalty-code-checks) * [Success message](#success-message) @@ -285,6 +286,39 @@ Meta test: returns wrong elements (weight: 1) FAILED Meta test: compares only elements in the same index (weight: 3) FAILED Meta test: background service does not work (weight: 2) PASSED ``` +#### Penalty meta tests + +Penalty meta tests work in a similar way to regular meta tests. However, they are not considered a grading component. Instead, if a penalty meta test passes, this has no effect on the final score. However, if it fails, its weight is subtracted from the final grade. + +Penalty meta tests must have a positive weight, but there is no upper limit on their total weight. For example, if there are two penalty meta tests where one of them has a weight of 10 and the other one 100 (see the example below), and a student fails both of them, the total applied penalty would be 110. In case this makes the final grade negative, the grade is reported as 0 instead. + +Penalty meta tests can be defined as follows: + +```java +@Override +public List penaltyMetaTests() { + return List.of( + MetaTest.insertAt(2, "returns empty list when list1 is null", 29, + "if (list1 == null) return result;" + ), + MetaTest.withStringReplacement("returns wrong elements", + "if (hashSet.contains(e))", + "if (!hashSet.contains(e))" + ), + MetaTest.withLineReplacement(3, "compares only elements in the same index", 29, 41, + """ + for (int i = 0; i < list1.size() && i < list2.size(); i++) { + if (list1.get(i).equals(list2.get(i))) { + result.add(list1.get(i)); + } + } + """ + ) + ); +} +``` + +If this method is not overridden, penalty meta tests are disabled. #### Code checks diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/config/RunConfiguration.java b/andy/src/main/java/nl/tudelft/cse1110/andy/config/RunConfiguration.java index 25af1fe04..bfc67e918 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/config/RunConfiguration.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/config/RunConfiguration.java @@ -45,6 +45,10 @@ public List metaTests() { return Collections.emptyList(); } + public List penaltyMetaTests() { + return Collections.emptyList(); + } + public List listOfMutants() { return DEFAULTS; } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java b/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java index 75af9acc6..ff7681e39 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java @@ -16,6 +16,8 @@ public class SecureExamRunConfiguration extends RunConfiguration { private final List listOfMutants; private final int numberOfMutationsToConsider; private final ExternalProcess externalProcess; + private final boolean skipJacoco; + private final boolean skipPitest; public SecureExamRunConfiguration(RunConfiguration runConfigurationToClone) { this.classesUnderTest = runConfigurationToClone.classesUnderTest(); @@ -23,6 +25,8 @@ public SecureExamRunConfiguration(RunConfiguration runConfigurationToClone) { this.numberOfMutationsToConsider = runConfigurationToClone.numberOfMutationsToConsider(); this.externalProcess = runConfigurationToClone.externalProcess(); this.successMessage = runConfigurationToClone.successMessage(); + this.skipJacoco = runConfigurationToClone.skipJacoco(); + this.skipPitest = runConfigurationToClone.skipPitest(); } public Mode mode() { @@ -60,5 +64,13 @@ public String successMessage() { return successMessage; } + @Override + public boolean skipJacoco() { + return this.skipJacoco; + } + @Override + public boolean skipPitest() { + return this.skipPitest; + } } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/mode/ModeActionSelector.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/mode/ModeActionSelector.java index c2532c474..1fe2877d5 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/mode/ModeActionSelector.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/mode/ModeActionSelector.java @@ -129,6 +129,7 @@ public static List fullMode() { new RunPenaltyCodeChecksStep(), new RunCodeChecksStep(), new KillExternalProcessStep(), + new RunPenaltyMetaTestsStep(), new RunMetaTestsStep() ); } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java new file mode 100644 index 000000000..6041fe20b --- /dev/null +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunPenaltyMetaTestsStep.java @@ -0,0 +1,62 @@ +package nl.tudelft.cse1110.andy.execution.step; + +import nl.tudelft.cse1110.andy.config.DirectoryConfiguration; +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; +import nl.tudelft.cse1110.andy.execution.Context.Context; +import nl.tudelft.cse1110.andy.execution.ExecutionStep; +import nl.tudelft.cse1110.andy.result.MetaTestResult; +import nl.tudelft.cse1110.andy.result.ResultBuilder; + +import java.util.ArrayList; +import java.util.List; + +public class RunPenaltyMetaTestsStep implements ExecutionStep { + + @Override + public void execute(Context ctx, ResultBuilder result) { + DirectoryConfiguration dirCfg = ctx.getDirectoryConfiguration(); + RunConfiguration runCfg = ctx.getRunConfiguration(); + + int score = 0; + int totalWeight = 0; + + ClassLoader currentClassLoader = Thread.currentThread().getContextClassLoader(); + + try { + + List metaTests = runCfg.penaltyMetaTests(); + List metaTestResults = new ArrayList<>(); + + for (MetaTest metaTest : metaTests) { + + boolean passesTheMetaTest = metaTest.execute(ctx, dirCfg, runCfg); + + if (passesTheMetaTest) { + score += metaTest.getWeight(); + } + + metaTestResults.add(new MetaTestResult(metaTest.getName(), metaTest.getWeight(), passesTheMetaTest)); + + totalWeight += metaTest.getWeight(); + } + + result.logPenaltyMetaTests(score, totalWeight, metaTestResults); + } catch (Exception ex) { + result.genericFailure(this, ex); + } finally { + /* restore the class loader to the one before meta tests */ + Thread.currentThread().setContextClassLoader(currentClassLoader); + } + } + + @Override + public boolean equals(Object other) { + return other instanceof RunPenaltyMetaTestsStep; + } + + @Override + public int hashCode() { + return super.hashCode(); + } +} diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java index 71dc41291..11918ed57 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/grade/GradeValues.java @@ -86,7 +86,7 @@ public GradeValues setPenalty(int penalty) { return this; } - public static GradeValues fromResults(CoverageResult coverageResults, CodeChecksResult codeCheckResults, MutationTestingResult mutationResults, MetaTestsResult metaTestResults, CodeChecksResult penaltyCodeCheckResults) { + public static GradeValues fromResults(CoverageResult coverageResults, CodeChecksResult codeCheckResults, MutationTestingResult mutationResults, MetaTestsResult metaTestResults, MetaTestsResult penaltyMetaTestResults, CodeChecksResult penaltyCodeCheckResults) { GradeValues grades = new GradeValues(); grades.setBranchGrade(coverageResults.getCoveredBranches(), coverageResults.getTotalNumberOfBranches()); grades.setCheckGrade(codeCheckResults.getNumberOfPassedChecks(), codeCheckResults.getTotalNumberOfChecks()); @@ -94,7 +94,8 @@ public static GradeValues fromResults(CoverageResult coverageResults, CodeChecks grades.setMetaGrade(metaTestResults.getPassedMetaTests(), metaTestResults.getTotalTests()); // penalty is equal to the sum of the weights of all failed penalty code checks - grades.setPenalty(penaltyCodeCheckResults.getCheckResults().stream().mapToInt(check -> check.passed() ? 0 : check.getWeight()).sum()); + grades.setPenalty(penaltyCodeCheckResults.getCheckResults().stream().mapToInt(check -> check.passed() ? 0 : check.getWeight()).sum() + + penaltyMetaTestResults.getMetaTestResults().stream().mapToInt(check -> check.succeeded() ? 0 : check.getWeight()).sum()); return grades; } diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java index 27356df70..18d0bf374 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/Result.java @@ -13,6 +13,7 @@ public class Result { private final CodeChecksResult penaltyCodeChecks; private final CoverageResult coverage; private final MetaTestsResult metaTests; + private final MetaTestsResult penaltyMetaTests; private final int penalty; private final int finalGrade; private final GenericFailure genericFailure; @@ -20,7 +21,7 @@ public class Result { private final GradeWeight weights; private final String successMessage; - public Result(CompilationResult compilation, UnitTestsResult tests, MutationTestingResult mutationTesting, CodeChecksResult codeChecks, CodeChecksResult penaltyCodeChecks, CoverageResult coverage, MetaTestsResult metaTests, int penalty, int finalGrade, GenericFailure genericFailure, double timeInSeconds, GradeWeight weights, String successMessage) { + public Result(CompilationResult compilation, UnitTestsResult tests, MutationTestingResult mutationTesting, CodeChecksResult codeChecks, CodeChecksResult penaltyCodeChecks, CoverageResult coverage, MetaTestsResult metaTests, MetaTestsResult penaltyMetaTests, int penalty, int finalGrade, GenericFailure genericFailure, double timeInSeconds, GradeWeight weights, String successMessage) { this.compilation = compilation; this.tests = tests; this.mutationTesting = mutationTesting; @@ -28,6 +29,7 @@ public Result(CompilationResult compilation, UnitTestsResult tests, MutationTest this.penaltyCodeChecks = penaltyCodeChecks; this.coverage = coverage; this.metaTests = metaTests; + this.penaltyMetaTests = penaltyMetaTests; this.penalty = penalty; this.finalGrade = finalGrade; this.genericFailure = genericFailure; @@ -40,7 +42,7 @@ public Result(CompilationResult compilation, UnitTestsResult tests, MutationTest } public Result(CompilationResult compilation, double timeInSeconds) { - this(compilation, UnitTestsResult.empty(), MutationTestingResult.empty(), CodeChecksResult.empty(), CodeChecksResult.empty(), CoverageResult.empty(), MetaTestsResult.empty(), 0, 0, GenericFailure.noFailure(), timeInSeconds, null, null); + this(compilation, UnitTestsResult.empty(), MutationTestingResult.empty(), CodeChecksResult.empty(), CodeChecksResult.empty(), CoverageResult.empty(), MetaTestsResult.empty(), MetaTestsResult.empty(), 0, 0, GenericFailure.noFailure(), timeInSeconds, null, null); } public CompilationResult getCompilation() { @@ -70,6 +72,9 @@ public CoverageResult getCoverage() { public MetaTestsResult getMetaTests() { return metaTests; } + public MetaTestsResult getPenaltyMetaTests() { + return penaltyMetaTests; + } public int getPenalty() { return penalty; diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index b2898c48f..35aff375f 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -49,6 +49,7 @@ public class ResultBuilder { private CodeChecksResult penaltyCodeCheckResults = CodeChecksResult.empty(); private CoverageResult coverageResults = CoverageResult.empty(); private MetaTestsResult metaTestResults = MetaTestsResult.empty(); + private MetaTestsResult penaltyMetaTestResults = MetaTestsResult.empty(); public ResultBuilder(Context ctx, GradeCalculator gradeCalculator) { this.ctx = ctx; @@ -244,6 +245,10 @@ public void logMetaTests(int score, int totalTests, List metaTes this.metaTestResults.addResults(score, totalTests, metaTestResults); } + public void logPenaltyMetaTests(int score, int totalTests, List metaTestResults) { + this.penaltyMetaTestResults.addResults(score, totalTests, metaTestResults); + } + /* * Generic failures */ @@ -271,7 +276,7 @@ public Result build() { if(!compilation.successful()) { return new Result(compilation, timeInSeconds); } else { - GradeValues grades = GradeValues.fromResults(coverageResults, codeCheckResults, mutationResults, metaTestResults, penaltyCodeCheckResults); + GradeValues grades = GradeValues.fromResults(coverageResults, codeCheckResults, mutationResults, metaTestResults, penaltyMetaTestResults, penaltyCodeCheckResults); GradeWeight weights = GradeWeight.fromConfig(ctx.getRunConfiguration().weights()); String successMessage = ctx.getRunConfiguration().successMessage(); @@ -287,6 +292,7 @@ public Result build() { penaltyCodeCheckResults, coverageResults, metaTestResults, + penaltyMetaTestResults, penalty, finalGrade, genericFailureObject, diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java b/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java index 811654428..bfedbf37a 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/writer/standard/StandardResultWriter.java @@ -65,7 +65,7 @@ private void writeStdOutFile(Context ctx, Result result) { printCoverageResults(result.getCoverage()); printMutationTestingResults(result.getMutationTesting()); printCodeCheckResults(ctx, result.getCodeChecks(), result.getPenaltyCodeChecks()); - printMetaTestResults(ctx, result.getMetaTests()); + printMetaTestResults(ctx, result.getMetaTests(), result.getPenaltyMetaTests()); printFinalGrade(ctx, result); printModeAndTimeToRun(ctx, result.getTimeInSeconds()); } @@ -244,8 +244,8 @@ private void printModeAndTimeToRun(Context ctx, double timeInSeconds) { l(String.format("\nAndy took %.1f seconds to assess your solution.", timeInSeconds)); } - private void printMetaTestResults(Context ctx, MetaTestsResult metaTests) { - if (!metaTests.wasExecuted() || metaTests.hasNoMetaTests()) + private void printMetaTestResults(Context ctx, MetaTestsResult metaTests, MetaTestsResult penaltyMetaTests) { + if ((!metaTests.wasExecuted() || metaTests.hasNoMetaTests()) && (!penaltyMetaTests.wasExecuted() || penaltyMetaTests.hasNoMetaTests())) return; boolean allHints = modeActionSelector(ctx).shouldShowFullHints(); @@ -255,7 +255,8 @@ private void printMetaTestResults(Context ctx, MetaTestsResult metaTests) { return; l("\n--- Meta tests"); - l(String.format("%d/%d passed", metaTests.getPassedMetaTests(), metaTests.getTotalTests())); + l(String.format("%d/%d passed", metaTests.getPassedMetaTests() + penaltyMetaTests.getPassedMetaTests(), + metaTests.getTotalTests() + penaltyMetaTests.getTotalTests())); if (allHints) { for (MetaTestResult metaTestResult : metaTests.getMetaTestResults()) { @@ -265,6 +266,13 @@ private void printMetaTestResults(Context ctx, MetaTestsResult metaTests) { l(String.format("Meta test: %s (weight: %d) FAILED", metaTestResult.getName(), metaTestResult.getWeight())); } } + for (MetaTestResult penaltyResult : penaltyMetaTests.getMetaTestResults()) { + if (penaltyResult.succeeded()) { + l(String.format("Meta test: %s (penalty: %d) PASSED", penaltyResult.getName(), penaltyResult.getWeight())); + } else { + l(String.format("Meta test: %s (penalty: %d) FAILED", penaltyResult.getName(), penaltyResult.getWeight())); + } + } } } diff --git a/andy/src/test/java/integration/LibraryMetaTestsTest.java b/andy/src/test/java/integration/LibraryMetaTestsTest.java index 3199ca88f..9faba1b6b 100644 --- a/andy/src/test/java/integration/LibraryMetaTestsTest.java +++ b/andy/src/test/java/integration/LibraryMetaTestsTest.java @@ -88,4 +88,46 @@ void metaTestInternalFailure() { .isEqualTo(RunMetaTestsStep.class.getSimpleName()); } + @Test + void allPenaltyMetaTestsPassing() { + Result result = run("NumberUtilsAddLibrary", + "NumberUtilsAddOfficialSolution", "NumberUtilsAddWithPenaltyMetaTestsConfiguration"); + + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests()) + .has(passedMetaTest("DoesNotApplyCarryAtAll")) + .has(passedMetaTest("DoesNotApplyLastCarry")) + .has(passedMetaTest("DoesNotCheckNumbersOutOfRange")); + } + + @Test + void somePenaltyMetaTestFailing() { + Result result = run("NumberUtilsAddLibrary", + "NumberUtilsAddAllTestsPass", "NumberUtilsAddWithPenaltyMetaTestsConfiguration"); + + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(1); + + assertThat(result.getPenaltyMetaTests()) + .has(passedMetaTest("DoesNotCheckNumbersOutOfRange")) + .has(failedMetaTest("DoesNotApplyCarryAtAll")) + .has(failedMetaTest("DoesNotApplyLastCarry")); + } + + @Test + void somePenaltyMetaTestFailingWithWeights() { + Result result = run("NumberUtilsAddLibrary", + "NumberUtilsAddAllTestsPass", "NumberUtilsAddConfigurationWithWeightWithPenaltyMetaTests"); + + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(7); + assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(2); + + assertThat(result.getPenaltyMetaTests()) + .has(passedMetaTest("DoesNotCheckNumbersOutOfRange")) + .has(failedMetaTest("AppliesMultipleCarriesWrongly")) + .has(failedMetaTest("DoesNotApplyCarryAtAll")) + .has(failedMetaTest("DoesNotApplyLastCarry")); + } + } \ No newline at end of file diff --git a/andy/src/test/java/integration/ModesAndActionsTest.java b/andy/src/test/java/integration/ModesAndActionsTest.java index 4a1c66ad6..38a262cd9 100644 --- a/andy/src/test/java/integration/ModesAndActionsTest.java +++ b/andy/src/test/java/integration/ModesAndActionsTest.java @@ -31,6 +31,8 @@ void practiceModeRunsEverything() { .has(codeCheck("getTripById should be set up", true, 1)); assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isTrue(); + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(0); assertThat(result.getMetaTests()).has(failedMetaTest("DoesNotCheckInvalidTripId")); assertThat(result.getFinalGrade()).isEqualTo(91); } @@ -49,10 +51,33 @@ void noPenaltyCodeChecksDefined() { assertThat(result.getPenaltyCodeChecks().wasExecuted()).isTrue(); assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isTrue(); + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(0); assertThat(result.getMetaTests()).has(failedMetaTest("DoesNotCheckInvalidTripId")); assertThat(result.getFinalGrade()).isEqualTo(91); } + @Test + void failingPenaltyMetaTests() { + Result result = run(Action.FULL_WITH_HINTS, "SoftWhereLibrary", "SoftWhereMissingTests", "SoftWhereConfigPenaltyMetaTestsFailing"); + + assertThat(result.getTests().getTestsSucceeded()).isEqualTo(2); + assertThat(result.getCoverage().getCoveredLines()).isEqualTo(11); + assertThat(result.getMutationTesting().getKilledMutants()).isEqualTo(8); + assertThat(result.getMutationTesting().getTotalNumberOfMutants()).isEqualTo(9); + assertThat(result.getCodeChecks().getNumberOfPassedChecks()).isEqualTo(3); + assertThat(result.getCodeChecks().getTotalNumberOfChecks()).isEqualTo(3); + assertThat(result.getPenaltyCodeChecks().hasChecks()).isFalse(); + assertThat(result.getPenaltyCodeChecks().wasExecuted()).isTrue(); + assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); + assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isTrue(); + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(20); + assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(0); + assertThat(result.getPenaltyMetaTests()).has(failedMetaTest("DoesNotCheckInvalidTripId")); + assertThat(result.getFinalGrade()).isEqualTo(91 - 20); + } + @Test void failingPenaltyCodeChecks() { Result result = run(Action.FULL_WITH_HINTS, "SoftWhereLibrary", "SoftWhereMissingTests", "SoftWhereConfigMetaAndCodeChecksPenaltyCodeChecksFailing"); @@ -67,6 +92,8 @@ void failingPenaltyCodeChecks() { assertThat(result.getPenaltyCodeChecks().wasExecuted()).isTrue(); assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isTrue(); + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(0); assertThat(result.getPenaltyCodeChecks().getNumberOfPassedChecks()).isEqualTo(10); assertThat(result.getPenaltyCodeChecks().getTotalNumberOfChecks()).isEqualTo(10 + 5 + 3); assertThat(result) @@ -76,6 +103,29 @@ void failingPenaltyCodeChecks() { assertThat(result.getFinalGrade()).isEqualTo(91 - (5 + 3)); } + @Test + void failingPenaltyMetaTestsOverrideGradeTo0() { + Result result = run(Action.FULL_WITH_HINTS, "SoftWhereLibrary", "SoftWhereMissingTests", "SoftWhereConfigPenaltyMetaTestsFailing2"); + + assertThat(result.getTests().getTestsSucceeded()).isEqualTo(2); + assertThat(result.getCoverage().getCoveredLines()).isEqualTo(11); + assertThat(result.getMutationTesting().getKilledMutants()).isEqualTo(8); + assertThat(result.getMutationTesting().getTotalNumberOfMutants()).isEqualTo(9); + assertThat(result.getCodeChecks().getNumberOfPassedChecks()).isEqualTo(3); + assertThat(result.getCodeChecks().getTotalNumberOfChecks()).isEqualTo(3); + assertThat(result.getPenaltyCodeChecks().hasChecks()).isFalse(); + assertThat(result.getPenaltyCodeChecks().wasExecuted()).isTrue(); + assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); + assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isTrue(); + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(220); + assertThat(result.getPenaltyMetaTests().getPassedMetaTests()).isEqualTo(0); + assertThat(result.getPenaltyMetaTests()).has(failedMetaTest("DoesNotCheckInvalidTripId")); + assertThat(result.getPenaltyMetaTests()).has(failedMetaTest("DoesNotCheckInvalidTripId2")); + assertThat(result.getFinalGrade()).isEqualTo(0); + } + + @Test void failingPenaltyCodeChecksOverrideGradeTo0() { Result result = run(Action.FULL_WITH_HINTS, "SoftWhereLibrary", "SoftWhereMissingTests", "SoftWhereConfigMetaAndCodeChecksPenaltyCodeChecksFailing2"); @@ -90,6 +140,8 @@ void failingPenaltyCodeChecksOverrideGradeTo0() { assertThat(result.getPenaltyCodeChecks().wasExecuted()).isTrue(); assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isTrue(); + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(0); assertThat(result.getPenaltyCodeChecks().getNumberOfPassedChecks(false)).isEqualTo(1); assertThat(result.getPenaltyCodeChecks().getTotalNumberOfChecks(false)).isEqualTo(3); assertThat(result) @@ -111,6 +163,7 @@ void runOnlyTests() { assertThat(result.getMutationTesting().wasExecuted()).isFalse(); assertThat(result.getCodeChecks().wasExecuted()).isFalse(); assertThat(result.getPenaltyCodeChecks().wasExecuted()).isFalse(); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isFalse(); assertThat(result.getFinalGrade()).isEqualTo(0); } @@ -130,6 +183,7 @@ void runOnlyTestsAndCoverageToolsDuringExam() { assertThat(result.getMutationTesting().getTotalNumberOfMutants()).isEqualTo(9); assertThat(result.getMetaTests().wasExecuted()).isFalse(); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isFalse(); assertThat(result.getCodeChecks().wasExecuted()).isFalse(); assertThat(result.getPenaltyCodeChecks().wasExecuted()).isFalse(); @@ -157,6 +211,8 @@ void gradingModeShouldRunEverything(Action action) { .has(penaltyCodeCheck("getTripById should not be set up penalty", false, 5)); assertThat(result.getMetaTests().getTotalTests()).isEqualTo(4); assertThat(result.getMetaTests().getPassedMetaTests()).isEqualTo(3); + assertThat(result.getPenaltyMetaTests().wasExecuted()).isTrue(); + assertThat(result.getPenaltyMetaTests().getTotalTests()).isEqualTo(0); assertThat(result.getMetaTests()).has(failedMetaTest("DoesNotCheckInvalidTripId")); assertThat(result.getFinalGrade()).isEqualTo(86); } diff --git a/andy/src/test/java/testutils/ResultTestDataBuilder.java b/andy/src/test/java/testutils/ResultTestDataBuilder.java index 1289b7c3c..a1c67e057 100644 --- a/andy/src/test/java/testutils/ResultTestDataBuilder.java +++ b/andy/src/test/java/testutils/ResultTestDataBuilder.java @@ -20,6 +20,7 @@ public class ResultTestDataBuilder { private CodeChecksResult penaltyCodeCheckResults = CodeChecksResult.empty(); private CoverageResult coverageResults = CoverageResult.empty(); private MetaTestsResult metaTestResults = MetaTestsResult.empty(); + private MetaTestsResult penaltyMetaTestResults = MetaTestsResult.empty(); private int penalty = 0; private int finalGrade = 0; private double timeInSeconds = 10; @@ -95,6 +96,15 @@ public ResultTestDataBuilder withMetaTestResults(List list) { return this; } + public ResultTestDataBuilder withPenaltyMetaTestResults(List list) { + penaltyMetaTestResults = MetaTestsResult.build( + (int) list.stream().filter(MetaTestResult::succeeded).count(), + list.size(), + list + ); + return this; + } + public ResultTestDataBuilder withWeights(float branchCoverageWeight, float mutationCoverageWeight, float metaTestsWeight, float codeChecksWeight) { weights = new GradeWeight(branchCoverageWeight, mutationCoverageWeight, metaTestsWeight, codeChecksWeight); return this; @@ -113,6 +123,6 @@ public ResultTestDataBuilder withSuccessMessage(String successMessage) { public Result build() { GenericFailure genericFailure = GenericFailure.build(genericFailureMessage, genericFailureStepName, genericFailureExceptionMessage, genericFailureExternalProcessExitCode, genericFailureExternalProcessErrorMessages); - return new Result(compilation, testResults, mutationResults, codeCheckResults, penaltyCodeCheckResults, coverageResults, metaTestResults, penalty, finalGrade, genericFailure, timeInSeconds, weights, successMessage); + return new Result(compilation, testResults, mutationResults, codeCheckResults, penaltyCodeCheckResults, coverageResults, metaTestResults, penaltyMetaTestResults, penalty, finalGrade, genericFailure, timeInSeconds, weights, successMessage); } } diff --git a/andy/src/test/java/unit/writer/standard/StandardResultTestAssertions.java b/andy/src/test/java/unit/writer/standard/StandardResultTestAssertions.java index d72664d9e..2605a11ac 100644 --- a/andy/src/test/java/unit/writer/standard/StandardResultTestAssertions.java +++ b/andy/src/test/java/unit/writer/standard/StandardResultTestAssertions.java @@ -171,6 +171,14 @@ public static Condition metaTestPassing(String metaTestName) { return containsRegex("Meta test: " + metaTestName + " \\(.*\\) PASSED"); } + public static Condition penaltyMetaTestFailing(String metaTestName) { + return containsRegex("Meta test: " + metaTestName + " \\(penalty: .*\\) FAILED"); + } + + public static Condition penaltyMetaTestPassing(String metaTestName) { + return containsRegex("Meta test: " + metaTestName + " \\(penalty: .*\\) PASSED"); + } + public static Condition finalGrade(int score) { return new Condition<>() { @Override @@ -327,7 +335,7 @@ public static Condition noCodeChecks() { return not(codeChecks()); } - public static Condition noPenaltyCodeChecks() { + public static Condition noPenaltyCodeChecksOrMetaTests() { return not(containsRegex(".*\\(penalty: \\d+\\)")); } diff --git a/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java b/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java index 60d506479..6f965efd8 100644 --- a/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java +++ b/andy/src/test/java/unit/writer/standard/StandardResultWriterTest.java @@ -284,7 +284,7 @@ void testPrintFinalGrade() { .has(mutationScore(5, 6)) .has(noMetaTests()) .has(noCodeChecks()) - .has(noPenaltyCodeChecks()) + .has(noPenaltyCodeChecksOrMetaTests()) .has(noPenalty()) .has(not(zeroScoreExplanation())) .doesNotContain("test success message"); @@ -570,6 +570,11 @@ void testPrintFinalGradeWithCodeChecksAndMetaTestsDisplayed(boolean fullHints, b new MetaTestResult("e", 3, false), new MetaTestResult("f", 1, true) )) + .withPenaltyMetaTestResults(List.of( + new MetaTestResult("d1", 16, true), + new MetaTestResult("e1", 32, false), + new MetaTestResult("f1", 64, true) + )) .withPenaltyCodeCheckResults(List.of( new CodeCheckResult("a1", 1, true), new CodeCheckResult("b1", 1, true), @@ -581,7 +586,6 @@ void testPrintFinalGradeWithCodeChecksAndMetaTestsDisplayed(boolean fullHints, b writer.write(ctx, result); String output = generatedResult(); - assertThat(output) .has(versionInformation(versionInformation)) .has(finalGradeOnScreen(34)) @@ -595,8 +599,8 @@ void testPrintFinalGradeWithCodeChecksAndMetaTestsDisplayed(boolean fullHints, b .has(fullGradeDescriptionDisplayed("Meta tests", 2, 3, 0.25)) .has(mutationScore(5, 6)) .has(scoreOfCodeChecks(7, 8)) - .has(metaTestsPassing(2)) - .has(metaTests(3)) + .has(metaTestsPassing(4)) + .has(metaTests(6)) .has(noPenalty()) .has(not(zeroScoreExplanation())); @@ -604,6 +608,9 @@ void testPrintFinalGradeWithCodeChecksAndMetaTestsDisplayed(boolean fullHints, b .has(metaTestDisplayed("d", true, fullHints)) .has(metaTestDisplayed("e", false, fullHints)) .has(metaTestDisplayed("f", true, fullHints)) + .has(penaltyMetaTestDisplayed("d1", true, fullHints)) + .has(penaltyMetaTestDisplayed("e1", false, fullHints)) + .has(penaltyMetaTestDisplayed("f1", true, fullHints)) .has(codeCheckDisplayed("a", true, 1, fullHints)) .has(codeCheckDisplayed("b", true, 2, fullHints)) .has(codeCheckDisplayed("c", false, 1, fullHints)) @@ -664,6 +671,55 @@ void testPrintFinalGradeWithPenaltyCodeChecksFailing(boolean fullHints, boolean .has(penaltyCodeCheckDisplayed("d1", true, 1, fullHints)); } + @ParameterizedTest + @CsvSource({ + "true,false", + "false,true", + "true,true" + }) + void testPrintFinalGradeWithPenaltyMetaTestsFailing(boolean fullHints, boolean partialHints) { + ModeActionSelector modeActionSelector = mock(ModeActionSelector.class); + when(modeActionSelector.shouldCalculateAndShowGrades()).thenReturn(true); + when(modeActionSelector.shouldGenerateAnalytics()).thenReturn(false); + when(modeActionSelector.shouldShowFullHints()).thenReturn(fullHints); + when(modeActionSelector.shouldShowPartialHints()).thenReturn(partialHints); + when(modeActionSelector.getMode()).thenReturn(Mode.PRACTICE); + + when(ctx.getModeActionSelector()).thenReturn(modeActionSelector); + + Result result = new ResultTestDataBuilder() + .withCoverageResult(CoverageResult.build( + 4, 7, 5, 8, 1, 2, + new CoverageLineByLine(List.of(), List.of(), List.of()))) + .withMutationTestingResults(5, 6) + .withPenaltyMetaTestResults(List.of( + new MetaTestResult("a1", 8, false), + new MetaTestResult("b1", 16, false), + new MetaTestResult("c1", 32, true), + new MetaTestResult("d1", 64, true) + )) + .withPenalty(24) + .build(); + + writer.write(ctx, result); + + String output = generatedResult(); + + assertThat(output) + .has(versionInformation(versionInformation)) + .has(compilationSuccess()) + .has(fullGradeDescriptionDisplayed("Meta tests", 0, 0, 0.25)) + .has(mutationScore(5, 6)) + .has(penalty(24)) + .has(not(zeroScoreExplanation())); + + assertThat(output) + .has(penaltyMetaTestDisplayed("a1", false, fullHints)) + .has(penaltyMetaTestDisplayed("b1", false, fullHints)) + .has(penaltyMetaTestDisplayed("c1", true, fullHints)) + .has(penaltyMetaTestDisplayed("d1", true, fullHints)); + } + @Test void testPrintFinalGradeWithCodeChecksAndMetaTestsDisplayedButNoCodeChecksOrTests() { ModeActionSelector modeActionSelector = mock(ModeActionSelector.class); @@ -865,5 +921,9 @@ protected Condition metaTestDisplayed(String description, boolea var metaTestCondition = pass ? metaTestPassing(description) : metaTestFailing(description); return shownInOutput ? metaTestCondition : not(metaTestCondition); } + protected Condition penaltyMetaTestDisplayed(String description, boolean pass, boolean shownInOutput) { + var penaltyMetaTestCondition = pass ? penaltyMetaTestPassing(description) : penaltyMetaTestFailing(description); + return shownInOutput ? penaltyMetaTestCondition : not(penaltyMetaTestCondition); + } } diff --git a/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddConfigurationWithWeightWithPenaltyMetaTests.java b/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddConfigurationWithWeightWithPenaltyMetaTests.java new file mode 100644 index 000000000..a8c3afd20 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddConfigurationWithWeightWithPenaltyMetaTests.java @@ -0,0 +1,100 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.RunConfiguration; +import nl.tudelft.cse1110.andy.config.MetaTest; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.9f); + put("meta", 0.0f); + put("codechecks", 0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.NumberUtils"); + } + + @Override + public List penaltyMetaTests() { + return List.of( + MetaTest.withStringReplacement(3, "AppliesMultipleCarriesWrongly", + """ + int sum = leftDigit + rightDigit + carry; + + result.addFirst(sum % 10); + carry = sum / 10; + """, + """ + int sum; + + if (leftDigit + rightDigit >= 10) { + sum = leftDigit + rightDigit; + carry = 1; + } + else { + sum = leftDigit + rightDigit + carry; + carry = 0; + } + result.addFirst(sum % 10); + """), + MetaTest.withLineReplacement("DoesNotApplyCarryAtAll", 47, 68, + """ + for (int i = 0; i < Math.max(reversedLeft.size(), reversedRight.size()); i++) { + + int leftDigit = reversedLeft.size() > i ? reversedLeft.get(i) : 0; + int rightDigit = reversedRight.size() > i ? reversedRight.get(i) : 0; + + if (leftDigit < 0 || leftDigit > 9 || rightDigit < 0 || rightDigit > 9) + throw new IllegalArgumentException(); + + int sum = leftDigit + rightDigit; + + result.addFirst(sum % 10); + } + + // remove leading zeroes from the result + while (result.size() > 1 && result.get(0) == 0) + result.remove(0); + + if (result.isEmpty()) { + result.addFirst(0); + } + + return result; + """), + MetaTest.withStringReplacement("DoesNotApplyLastCarry", + """ + // add leftover carry + result.addFirst(carry); + + // remove leading zeroes from the result + while (result.size() > 1 && result.get(0) == 0) + result.remove(0); + + return result; + """, + """ + // remove leading zeroes from the result + while (result.size() > 1 && result.get(0) == 0) + result.remove(0); + + if (result.isEmpty()) { + result.addFirst(0); + } + + return result; + """), + MetaTest.withLineReplacement(2, "DoesNotCheckNumbersOutOfRange", 52, 53, "") + ); + } +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddWithPenaltyMetaTestsConfiguration.java b/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddWithPenaltyMetaTestsConfiguration.java new file mode 100644 index 000000000..228399839 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddWithPenaltyMetaTestsConfiguration.java @@ -0,0 +1,105 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.RunConfiguration; +import nl.tudelft.cse1110.andy.config.MetaTest; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.7f); + put("meta", 0.2f); + put("codechecks", 0f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.NumberUtils"); + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withStringReplacement("AppliesMultipleCarriesWrongly", + """ + int sum = leftDigit + rightDigit + carry; + + result.addFirst(sum % 10); + carry = sum / 10; + """, + """ + int sum; + + if (leftDigit + rightDigit >= 10) { + sum = leftDigit + rightDigit; + carry = 1; + } + else { + sum = leftDigit + rightDigit + carry; + carry = 0; + } + result.addFirst(sum % 10); + """) + ); + } + @Override + public List penaltyMetaTests() { + return List.of( + MetaTest.withLineReplacement("DoesNotApplyCarryAtAll", 47, 68, + """ + for (int i = 0; i < Math.max(reversedLeft.size(), reversedRight.size()); i++) { + + int leftDigit = reversedLeft.size() > i ? reversedLeft.get(i) : 0; + int rightDigit = reversedRight.size() > i ? reversedRight.get(i) : 0; + + if (leftDigit < 0 || leftDigit > 9 || rightDigit < 0 || rightDigit > 9) + throw new IllegalArgumentException(); + + int sum = leftDigit + rightDigit; + + result.addFirst(sum % 10); + } + + // remove leading zeroes from the result + while (result.size() > 1 && result.get(0) == 0) + result.remove(0); + + if (result.isEmpty()) { + result.addFirst(0); + } + + return result; + """), + MetaTest.withStringReplacement("DoesNotApplyLastCarry", + """ + // add leftover carry + result.addFirst(carry); + + // remove leading zeroes from the result + while (result.size() > 1 && result.get(0) == 0) + result.remove(0); + + return result; + """, + """ + // remove leading zeroes from the result + while (result.size() > 1 && result.get(0) == 0) + result.remove(0); + + if (result.isEmpty()) { + result.addFirst(0); + } + + return result; + """), + MetaTest.withLineReplacement("DoesNotCheckNumbersOutOfRange", 52, 53, "") + ); + } +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigPenaltyMetaTestsFailing.java b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigPenaltyMetaTestsFailing.java new file mode 100644 index 000000000..80ca5b051 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigPenaltyMetaTestsFailing.java @@ -0,0 +1,90 @@ +package delft; + +import nl.tudelft.cse1110.andy.codechecker.checks.Comparison; +import nl.tudelft.cse1110.andy.codechecker.checks.MockClass; +import nl.tudelft.cse1110.andy.codechecker.checks.MockitoWhen; +import nl.tudelft.cse1110.andy.codechecker.engine.CheckScript; +import nl.tudelft.cse1110.andy.codechecker.engine.SingleCheck; +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.25f); + put("mutation", 0.25f); + put("meta", 0.25f); + put("codechecks", 0.25f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.SoftWhere"); + } + + @Override + public CheckScript checkScript() { + return new CheckScript(List.of( + new SingleCheck("Trip Repository should be mocked", new MockClass("TripRepository")), + new SingleCheck( "Trip should not be mocked", true, new MockClass("Trip")), + new SingleCheck( "getTripById should be set up", new MockitoWhen("getTripById", Comparison.GTE, 1)) + )); + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withLineReplacement("DoesNotCheckInvalidTripId", 150, 158, + """ + try { + Trip trip = tRepository.getTripById(tripId); + if (capacityLeft(trip) < people.size()) return false; + rRepository.save(new Reservation(trip, people)); + return true; + } catch (ElementNotFoundException e) { + throw new RuntimeException("killed the mutant"); + } + """), + MetaTest.withStringReplacement("BoundaryCheck", + """ + if (capacityLeft(trip) < people.size()) + return false; + """, + """ + if (capacityLeft(trip) <= people.size()) + return false; + """), + MetaTest.withStringReplacement("DoesNotCheckCapacity", + """ + if (capacityLeft(trip) < people.size()) + return false; + """,""), + MetaTest.withLineReplacement("DoesNotCheckSave", 154, 154, "") + ); + } + + @Override + public List penaltyMetaTests() { + return List.of( + MetaTest.withLineReplacement(20,"DoesNotCheckInvalidTripId", 150, 158, + """ + try { + Trip trip = tRepository.getTripById(tripId); + if (capacityLeft(trip) < people.size()) return false; + rRepository.save(new Reservation(trip, people)); + return true; + } catch (ElementNotFoundException e) { + throw new RuntimeException("killed the mutant"); + } + """) + ); + } + +} \ No newline at end of file diff --git a/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigPenaltyMetaTestsFailing2.java b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigPenaltyMetaTestsFailing2.java new file mode 100644 index 000000000..150d4e4ab --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/SoftWhereConfigPenaltyMetaTestsFailing2.java @@ -0,0 +1,102 @@ +package delft; + +import nl.tudelft.cse1110.andy.codechecker.checks.Comparison; +import nl.tudelft.cse1110.andy.codechecker.checks.MockClass; +import nl.tudelft.cse1110.andy.codechecker.checks.MockitoWhen; +import nl.tudelft.cse1110.andy.codechecker.engine.CheckScript; +import nl.tudelft.cse1110.andy.codechecker.engine.SingleCheck; +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.config.RunConfiguration; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends RunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.25f); + put("mutation", 0.25f); + put("meta", 0.25f); + put("codechecks", 0.25f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.SoftWhere"); + } + + @Override + public CheckScript checkScript() { + return new CheckScript(List.of( + new SingleCheck("Trip Repository should be mocked", new MockClass("TripRepository")), + new SingleCheck( "Trip should not be mocked", true, new MockClass("Trip")), + new SingleCheck( "getTripById should be set up", new MockitoWhen("getTripById", Comparison.GTE, 1)) + )); + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withLineReplacement("DoesNotCheckInvalidTripId", 150, 158, + """ + try { + Trip trip = tRepository.getTripById(tripId); + if (capacityLeft(trip) < people.size()) return false; + rRepository.save(new Reservation(trip, people)); + return true; + } catch (ElementNotFoundException e) { + throw new RuntimeException("killed the mutant"); + } + """), + MetaTest.withStringReplacement("BoundaryCheck", + """ + if (capacityLeft(trip) < people.size()) + return false; + """, + """ + if (capacityLeft(trip) <= people.size()) + return false; + """), + MetaTest.withStringReplacement("DoesNotCheckCapacity", + """ + if (capacityLeft(trip) < people.size()) + return false; + """,""), + MetaTest.withLineReplacement("DoesNotCheckSave", 154, 154, "") + ); + } + + @Override + public List penaltyMetaTests() { + return List.of( + MetaTest.withLineReplacement(20,"DoesNotCheckInvalidTripId", 150, 158, + """ + try { + Trip trip = tRepository.getTripById(tripId); + if (capacityLeft(trip) < people.size()) return false; + rRepository.save(new Reservation(trip, people)); + return true; + } catch (ElementNotFoundException e) { + throw new RuntimeException("killed the mutant"); + } + """), + + MetaTest.withLineReplacement(200,"DoesNotCheckInvalidTripId2", 150, 158, + """ + try { + Trip trip = tRepository.getTripById(tripId); + if (capacityLeft(trip) < people.size()) return false; + rRepository.save(new Reservation(trip, people)); + return true; + } catch (ElementNotFoundException e) { + throw new RuntimeException("killed the mutant"); + } + """) + ); + } + +} \ No newline at end of file diff --git a/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java b/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java index 047fce61b..b3e49f973 100644 --- a/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java +++ b/weblab-runner/src/main/java/nl/tudelft/cse1110/andy/writer/weblab/WebLabResultWriter.java @@ -102,6 +102,7 @@ private static void appendMetaScoreElements(Result result, Document doc, Element result.getCodeChecks().getCheckResults().forEach(check -> appendMetaScore(doc, metaElement, check.getDescription(), check.passed() ? 1 : 0)); result.getPenaltyCodeChecks().getCheckResults().forEach(check -> appendMetaScore(doc, metaElement, check.getDescription(), check.passed() ? 1 : 0)); result.getMetaTests().getMetaTestResults().forEach(metaTest -> appendMetaScore(doc, metaElement, metaTest.getName(), metaTest.succeeded() ? 1 : 0)); + result.getPenaltyMetaTests().getMetaTestResults().forEach(metaTest -> appendMetaScore(doc, metaElement, metaTest.getName(), metaTest.succeeded() ? 1 : 0)); testSuitesElement.appendChild(metaElement); }