Move autoremove builds logic to a scheduled job

.env.example

@@ -25,6 +25,15 @@ DB_PASSWORD=secret
 # Should CDash automatically remove old builds?
 #AUTOREMOVE_BUILDS=true
 
+# How many builds should CDash try to remove per iteration?
+# Consider tweaking this value if you notice the autoremove queries
+# are slow to execute.
+#AUTOREMOVE_BUILDS_BATCH_SIZE=10
+
+# Skip the autoremove builds step if more than this number of submissions
+# are waiting to be parsed.
+#AUTOREMOVE_BUILDS_SKIP_THRESHOLD=100
+
 # How long should CDash store parsed input files (in hours?)
 # Set to 0 if you do not wish to backup parsed submission files.
 #BACKUP_TIMEFRAME=48

app/Console/Commands/CleanupDatabase.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace App\Console\Commands;
+
+use Illuminate\Console\Command;
+use Illuminate\Support\Facades\DB;
+
+class CleanupDatabase extends Command
+{
+    /**
+     * The name and signature of the console command.
+     */
+    protected $signature = 'db:cleanup';
+
+    /**
+     * The console command description.
+     */
+    protected $description = 'Delete orphaned records from the CDash database';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle(): void
+    {
+        self::delete_unused_rows('banner', 'projectid', 'project');
+        self::delete_unused_rows('build', 'projectid', 'project');
+        self::delete_unused_rows('dailyupdate', 'projectid', 'project');
+
+        self::delete_unused_rows('buildfailuredetails', 'id', 'buildfailure', 'detailsid');
+        self::delete_unused_rows('configure', 'id', 'build2configure', 'configureid');
+        self::delete_unused_rows('configureerror', 'configureid', 'configure');
+        self::delete_unused_rows('dailyupdatefile', 'dailyupdateid', 'dailyupdate');
+        self::delete_unused_rows('note', 'id', 'build2note', 'noteid');
+        self::delete_unused_rows('testoutput', 'id', 'build2test', 'outputid');
+        self::delete_unused_rows('updatefile', 'updateid', 'buildupdate');
+        self::delete_unused_rows('uploadfile', 'id', 'build2uploadfile', 'fileid');
+
+        self::delete_unused_rows('subproject2subproject', 'subprojectid', 'subproject');
+
+        self::delete_unused_rows('coveragefile', 'id', 'coverage', 'fileid');
+        self::delete_unused_rows('coveragefile2user', 'fileid', 'coveragefile');
+
+        self::delete_unused_rows('test2image', 'outputid', 'testoutput');
+        self::delete_unused_rows('image', 'id', 'test2image', 'imgid');
+    }
+
+    /** Delete unused rows */
+    private static function delete_unused_rows(string $table, string $field, string $targettable, string $selectfield = 'id'): void
+    {
+        DB::delete("DELETE FROM $table WHERE $field NOT IN (SELECT $selectfield AS $field FROM $targettable)");
+    }
+}

app/Console/Kernel.php

@@ -2,6 +2,7 @@
 
 namespace App\Console;
 
+use App\Jobs\CleanupDatabase;
 use App\Jobs\PruneAuthTokens;
 use App\Jobs\PruneJobs;
 use Illuminate\Console\Scheduling\Schedule;
@@ -25,14 +26,18 @@ protected function schedule(Schedule $schedule)
             ->everySixHours()
             ->sendOutputTo($output_filename);
 
-        $schedule->job(new PruneJobs())
-            ->hourly()
+        $schedule->job(new CleanupDatabase())
+            ->everyFourHours()
             ->withoutOverlapping();
 
         $schedule->job(new PruneAuthTokens())
             ->hourly()
             ->withoutOverlapping();
 
+        $schedule->job(new PruneJobs())
+            ->hourly()
+            ->withoutOverlapping();
+
         if (env('CDASH_AUTHENTICATION_PROVIDER') === 'ldap') {
             $schedule->command('ldap:sync_projects')
                 ->everyFiveMinutes();

app/Http/Controllers/AdminController.php

@@ -2,6 +2,7 @@
 namespace App\Http\Controllers;
 
 use App\Models\User;
+use App\Utils\DatabaseCleanupUtils;
 use CDash\Model\Project;
 use Illuminate\Http\RedirectResponse;
 use Illuminate\Support\Facades\DB;
@@ -96,7 +97,7 @@ public function removeBuilds(): View|RedirectResponse
                 $builds[] = (int) $build_array->id;
             }
 
-            remove_build_chunked($builds);
+            DatabaseCleanupUtils::removeBuildChunked($builds);
             $alert = 'Removed ' . count($builds) . ' builds.';
         }
 
@@ -330,7 +331,6 @@ public function upgrade()
         @$ComputeTestTiming = $_POST['ComputeTestTiming'];
         @$ComputeUpdateStatistics = $_POST['ComputeUpdateStatistics'];
 
-        @$Cleanup = $_POST['Cleanup'];
         @$Dependencies = $_POST['Dependencies'];
         @$Audit = $_POST['Audit'];
         @$ClearAudit = $_POST['Clear'];
@@ -376,57 +376,6 @@ public function upgrade()
             unlink($configFile);
         }
 
-
-        /* Cleanup the database */
-        if ($Cleanup) {
-            self::delete_unused_rows('banner', 'projectid', 'project');
-            self::delete_unused_rows('blockbuild', 'projectid', 'project');
-            self::delete_unused_rows('build', 'projectid', 'project');
-            self::delete_unused_rows('buildgroup', 'projectid', 'project');
-            self::delete_unused_rows('labelemail', 'projectid', 'project');
-            self::delete_unused_rows('project2repositories', 'projectid', 'project');
-            self::delete_unused_rows('dailyupdate', 'projectid', 'project');
-            self::delete_unused_rows('subproject', 'projectid', 'project');
-            self::delete_unused_rows('coveragefilepriority', 'projectid', 'project');
-            self::delete_unused_rows('user2project', 'projectid', 'project');
-            self::delete_unused_rows('userstatistics', 'projectid', 'project');
-
-            self::delete_unused_rows('build2configure', 'buildid', 'build');
-            self::delete_unused_rows('build2note', 'buildid', 'build');
-            self::delete_unused_rows('build2test', 'buildid', 'build');
-            self::delete_unused_rows('buildemail', 'buildid', 'build');
-            self::delete_unused_rows('builderror', 'buildid', 'build');
-            self::delete_unused_rows('builderrordiff', 'buildid', 'build');
-            self::delete_unused_rows('buildfailure', 'buildid', 'build');
-            self::delete_unused_rows('buildfailuredetails', 'id', 'buildfailure', 'detailsid');
-            self::delete_unused_rows('buildinformation', 'buildid', 'build');
-            self::delete_unused_rows('buildtesttime', 'buildid', 'build');
-            self::delete_unused_rows('configure', 'id', 'build2configure', 'configureid');
-            self::delete_unused_rows('configureerror', 'configureid', 'configure');
-            self::delete_unused_rows('configureerrordiff', 'buildid', 'build');
-            self::delete_unused_rows('coverage', 'buildid', 'build');
-            self::delete_unused_rows('coveragefilelog', 'buildid', 'build');
-            self::delete_unused_rows('coveragesummary', 'buildid', 'build');
-            self::delete_unused_rows('coveragesummarydiff', 'buildid', 'build');
-            self::delete_unused_rows('dynamicanalysis', 'buildid', 'build');
-            self::delete_unused_rows('label2build', 'buildid', 'build');
-            self::delete_unused_rows('subproject2build', 'buildid', 'build');
-            self::delete_unused_rows('summaryemail', 'buildid', 'build');
-            self::delete_unused_rows('testdiff', 'buildid', 'build');
-
-            self::delete_unused_rows('dynamicanalysisdefect', 'dynamicanalysisid', 'dynamicanalysis');
-            self::delete_unused_rows('subproject2subproject', 'subprojectid', 'subproject');
-
-            self::delete_unused_rows('dailyupdatefile', 'dailyupdateid', 'dailyupdate');
-            self::delete_unused_rows('coveragefile', 'id', 'coverage', 'fileid');
-            self::delete_unused_rows('coveragefile2user', 'fileid', 'coveragefile');
-
-            self::delete_unused_rows('dailyupdatefile', 'dailyupdateid', 'dailyupdate');
-            self::delete_unused_rows('test2image', 'outputid', 'testoutput');
-
-            $xml .= add_XML_value('alert', 'Database cleanup complete.');
-        }
-
         /* Check the builds with wrong date */
         if ($CheckBuildsWrongDate) {
             $currentdate = time() + 3600 * 24 * 3; // or 3 days away from now
@@ -448,7 +397,7 @@ public function upgrade()
           starttime<'1975-12-31 23:59:59' OR starttime>'$forwarddate'");
             while ($builds_array = pdo_fetch_array($builds)) {
                 $buildid = $builds_array['id'];
-                remove_build($buildid);
+                DatabaseCleanupUtils::removeBuild($buildid);
             }
         }
 
@@ -507,11 +456,4 @@ public function userStatistics(): View
     {
         return $this->angular_view('userStatistics');
     }
-
-    /** Delete unused rows */
-    private static function delete_unused_rows($table, $field, $targettable, $selectfield = 'id'): void
-    {
-        DB::delete("DELETE FROM $table WHERE $field NOT IN (SELECT $selectfield AS $field FROM $targettable)");
-        echo pdo_error();
-    }
 }

app/Http/Controllers/BuildController.php

@@ -4,6 +4,7 @@
 use App\Models\Comment;
 use App\Models\User;
 use App\Models\Build as EloquentBuild;
+use App\Utils\DatabaseCleanupUtils;
 use App\Utils\PageTimer;
 use App\Utils\RepositoryUtils;
 use App\Utils\TestingDay;
@@ -1471,7 +1472,7 @@ private function restApiPost(): JsonResponse
     private function restApiDelete(): JsonResponse
     {
         Log::info("Build #{$this->build->Id} removed manually.");
-        remove_build($this->build->Id);
+        DatabaseCleanupUtils::removeBuild($this->build->Id);
         return response()->json();
     }
 }

app/Jobs/CleanupDatabase.php

@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Jobs;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Artisan;
+use Illuminate\Support\Facades\DB;
+
+/**
+ * Removes builds that have expired according to per-project and
+ * per-buildgroup settings.
+ */
+class CleanupDatabase implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        if (!(bool) config('cdash.autoremove_builds')) {
+            return;
+        }
+
+        $skip_threshold = (int) config('cdash.autoremove_builds_skip_threshold');
+        if (DB::table('jobs')->count() > $skip_threshold) {
+            return;
+        }
+
+        Artisan::call('build:remove all');
+        Artisan::call('db:cleanup');
+    }
+}

app/Utils/DatabaseCleanupUtils.php

@@ -6,6 +6,7 @@
 
 use App\Models\Build;
 use App\Models\BuildGroup;
+use CDash\Database;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
 
@@ -54,7 +55,7 @@ public static function removeBuildsGroupwise(int $projectid, int $maxbuilds, boo
         $s = 'removing old buildids for projectid: ' . $projectid;
         Log::info($s);
         echo '  -- ' . $s . "\n";
-        remove_build_chunked($buildids);
+        self::removeBuildChunked($buildids);
     }
 
     /** Remove the first builds that are at the beginning of the queue */
@@ -89,6 +90,82 @@ public static function removeFirstBuilds(int $projectid, int $days, int $maxbuil
         if ($echo) {
             echo '  -- ' . $s . "\n"; // for "interactive" command line feedback
         }
-        remove_build_chunked($buildids);
+        self::removeBuildChunked($buildids);
+    }
+
+    /**
+     * Remove all related inserts for a given build or any build in an array of builds
+     * @param array<int>|int $buildid
+     * @throws \InvalidArgumentException
+     */
+    public static function removeBuild($buildid) : void
+    {
+        // TODO: (williamjallen) much of this work could be done on the DB side automatically by setting up
+        //       proper foreign-key relationships between between entities, and using the DB's cascade functionality.
+        //       For complex cascades, custom SQL functions can be written.
+
+        if (!is_array($buildid)) {
+            $buildid = [$buildid];
+        }
+
+        $buildids = [];
+        foreach ($buildid as $b) {
+            if (!is_numeric($b)) {
+                throw new \InvalidArgumentException('Invalid Build ID');
+            }
+            $buildids[] = intval($b);
+        }
+
+        $db = Database::getInstance();
+        $buildid_prepare_array = $db->createPreparedArray(count($buildids));
+
+        // Remove the buildfailureargument
+        $buildfailureids = [];
+        $buildfailure = DB::select("SELECT id FROM buildfailure WHERE buildid IN $buildid_prepare_array", $buildids);
+        foreach ($buildfailure as $buildfailure_array) {
+            $buildfailureids[] = intval($buildfailure_array->id);
+        }
+        if (count($buildfailureids) > 0) {
+            $buildfailure_prepare_array = $db->createPreparedArray(count($buildfailureids));
+            DB::delete("DELETE FROM buildfailure2argument WHERE buildfailureid IN $buildfailure_prepare_array", $buildfailureids);
+        }
+
+        // Remove any children of these builds.
+        // In order to avoid making the list of builds to delete too large
+        // we delete them in batches (one batch per parent).
+        foreach ($buildids as $parentid) {
+            $child_result = DB::select('SELECT id FROM build WHERE parentid=?', [intval($parentid)]);
+
+            $childids = [];
+            foreach ($child_result as $child_array) {
+                $childids[] = intval($child_array->id);
+            }
+            if (!empty($childids)) {
+                self::removeBuildChunked($childids);
+            }
+        }
+
+        // Only delete the buildid at the end so that no other build can get it in the meantime
+        DB::delete("DELETE FROM build WHERE id IN $buildid_prepare_array", $buildids);
+    }
+
+    /**
+     * Call removeBuild() in batches.
+     * @param array<int>|int $buildid
+     */
+    public static function removeBuildChunked($buildid): void
+    {
+        if (!is_array($buildid)) {
+            self::removeBuild($buildid);
+        }
+
+        $batch_size = (int) config('cdash.autoremove_builds_batch_size');
+        if ($batch_size < 1) {
+            $batch_size = 1;
+        }
+        foreach (array_chunk($buildid, $batch_size) as $chunk) {
+            self::removeBuild($chunk);
+            usleep(1);
+        }
     }
 }

app/Utils/SubmissionUtils.php

@@ -91,7 +91,7 @@ public static function add_build(Build $build)
         if ($buildid > 0 && !$build->Append) {
             $build->Id = $buildid;
             if ($build->GetDone()) {
-                remove_build($buildid);
+                DatabaseCleanupUtils::removeBuild($buildid);
                 $build->Id = null;
             }
         }

app/cdash/app/Model/Build.php

@@ -18,6 +18,7 @@
 
 use App\Models\Test;
 use App\Models\Site;
+use App\Utils\DatabaseCleanupUtils;
 use App\Utils\RepositoryUtils;
 use App\Utils\SubmissionUtils;
 use App\Utils\TestingDay;
@@ -2166,7 +2167,7 @@ public function RemoveIfDone(): bool
             return false;
         }
 
-        remove_build($this->Id);
+        DatabaseCleanupUtils::removeBuild($this->Id);
         $this->Id = 0;
         return true;
     }