Adding background jobs

.devcontainer/docker-compose.mysql.yml

@@ -18,6 +18,7 @@ services:
       # origin instead of a wildcard, so the dev frontend on :4200 has to be named here.
       HASHTOPOLIS_BACKEND_URL: http://localhost:8080/api/v2
       HASHTOPOLIS_FRONTEND_URLS: http://localhost:4200,http://127.0.0.1:4200
+      HASHTOPOLIS_CRON_ENABLE: 0
     depends_on:
       - hashtopolis-db-dev
     ports:

.devcontainer/docker-compose.postgres.yml

@@ -18,6 +18,7 @@ services:
       # origin instead of a wildcard, so the dev frontend on :4200 has to be named here.
       HASHTOPOLIS_BACKEND_URL: http://localhost:8080/api/v2
       HASHTOPOLIS_FRONTEND_URLS: http://localhost:4200,http://127.0.0.1:4200
+      HASHTOPOLIS_CRON_ENABLE: 0
     depends_on:
       - hashtopolis-db-dev
     ports:

.github/docker-compose.mysql.yml

@@ -16,6 +16,9 @@ services:
       HASHTOPOLIS_DB_DATABASE: hashtopolis
       HASHTOPOLIS_APIV2_ENABLE: 1
       HASHTOPOLIS_BACKEND_URL: http://localhost:80/api/v2
+      # The background job runner is invoked explicitly by the test suite for deterministic
+      # results, therefore the entrypoint scheduler loop is disabled in the test container.
+      HASHTOPOLIS_CRON_ENABLE: 0
     depends_on:
       - hashtopolis-db-dev
     ports:

.github/docker-compose.postgres.yml

@@ -16,6 +16,9 @@ services:
       HASHTOPOLIS_DB_DATABASE: hashtopolis
       HASHTOPOLIS_APIV2_ENABLE: 1
       HASHTOPOLIS_BACKEND_URL: http://localhost:80/api/v2
+      # The background job runner is invoked explicitly by the test suite for deterministic
+      # results, therefore the entrypoint scheduler loop is disabled in the test container.
+      HASHTOPOLIS_CRON_ENABLE: 0
     depends_on:
       - hashtopolis-db-dev
     ports:

ci/apiv2/test_backgroundjob.py

@@ -0,0 +1,119 @@
+import subprocess
+
+import requests
+
+from hashtopolis import File, Helper
+from utils import BaseTest, BackgroundJob, create_restricted_user, get_hashtopolis_uri
+
+CRON_RUNNER = '/var/www/html/src/inc/cron.php'
+
+
+def run_background_job_runner():
+    result = subprocess.run(
+        ['php', '-d', 'xdebug.mode=off', '-f', CRON_RUNNER],
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == 0, f"Background job runner failed: {result.stderr}"
+
+
+def get_jobs_for_file(file_id):
+    return [job for job in BackgroundJob.objects.all() if job.payload.get('fileId') == file_id]
+
+
+def get_admin_connector():
+    connector = BackgroundJob.objects.get_conn()
+    connector.authenticate()
+    return connector
+
+
+def enqueue_recount_job(file_obj):
+    helper = Helper()
+    helper.recount_file_lines(file=file_obj)
+
+
+class BackgroundJobTest(BaseTest):
+    model_class = BackgroundJob
+
+    def create_test_object(self, *nargs, delete=True, **kwargs):
+        # Background jobs cannot be created directly via the API, they get enqueued by
+        # other operations. The recount file lines helper is the reference producer.
+        file_obj = self.create_file()
+        enqueue_recount_job(file_obj)
+        jobs = get_jobs_for_file(file_obj.id)
+        assert len(jobs) == 1
+        if delete:
+            self.delete_after_test(jobs[0])
+        return jobs[0]
+
+    def test_recount_file_job_is_enqueued_as_pending(self):
+        model_obj = self.create_test_object()
+
+        self.assertEqual(model_obj.jobType, 'recount_file')
+        self.assertEqual(model_obj.status, 0)
+        self.assertIn('fileId', model_obj.payload)
+        self.assertIsNotNone(model_obj.userId)
+        self.assertIsNotNone(model_obj.createdAt)
+        self.assertIsNone(model_obj.startedAt)
+        self.assertIsNone(model_obj.finishedAt)
+
+    def test_recount_file_job_is_processed_by_runner(self):
+        model_obj = self.create_test_object()
+        file_id = model_obj.payload['fileId']
+
+        run_background_job_runner()
+
+        job = self.model_class.objects.get(pk=model_obj.id)
+        self.assertEqual(job.status, 2)
+        self.assertEqual(job.exitCode, 0)
+        self.assertEqual(job.resultMessage, 'Recounted 3 lines.')
+        self.assertIsNotNone(job.startedAt)
+        self.assertIsNotNone(job.finishedAt)
+
+        file = File.objects.get(pk=file_id)
+        self.assertEqual(file.lineCount, 3)
+
+    def test_delete_pending_job(self):
+        model_obj = self.create_test_object(delete=False)
+
+        # A pending (queued) job can be deleted, which cancels it
+        model_obj.delete()
+
+        with self.assertRaises(self.model_class.DoesNotExist):
+            _ = self.model_class.objects.get(pk=model_obj.id)
+
+    def test_delete_processed_job(self):
+        model_obj = self.create_test_object(delete=False)
+
+        run_background_job_runner()
+        model_obj.delete()
+
+        with self.assertRaises(self.model_class.DoesNotExist):
+            _ = self.model_class.objects.get(pk=model_obj.id)
+
+    def test_create_via_api_not_allowed(self):
+        connector = get_admin_connector()
+        response = requests.post(
+            connector._api_endpoint + '/ui/backgroundJobs',
+            headers=connector._headers,
+            data='{"data": {"type": "backgroundJob", "attributes": {}}}',
+        )
+        self.assertEqual(response.status_code, 405, response.text)
+
+    def test_restricted_user_cannot_access_without_server_config_access(self):
+        self.create_test_object()
+
+        auth = create_restricted_user(self, {'permFileRead': True})
+        response = requests.post(
+            get_hashtopolis_uri() + '/api/v2/auth/token',
+            auth=auth,
+        )
+        self.assertIn(response.status_code, [200, 201], response.text)
+        token = response.json()['token']
+
+        connector = get_admin_connector()
+        response = requests.get(
+            connector._api_endpoint + '/ui/backgroundJobs',
+            headers={'Authorization': f'Bearer {token}'},
+        )
+        self.assertEqual(response.status_code, 403, response.text)

ci/apiv2/test_file.py

@@ -1,7 +1,7 @@
 import pytest
 
 from hashtopolis import File, Helper
-from utils import BaseTest
+from utils import BaseTest, BackgroundJob
 
 
 class FileTest(BaseTest):
@@ -32,16 +32,21 @@ def test_create_binary(self):
         self._test_create(model_obj)
 
     def test_recount_wordlist(self):
-        # Note: After the object creation, the line count is already updated, but afterward it is immutable on the API.
-        #       There the test just check that the API function is callable and returns the file, but the count is
-        #       already the same beforehand.
+        # Note: The recounting is enqueued as background job, but it was already done on the object creation,
+        #       the file is returned with its immutable line count.
         model_obj = self.create_test_object()
 
         helper = Helper()
         file = helper.recount_file_lines(file=model_obj)
 
         self.assertEqual(file.lineCount, 3)
 
+        # The enqueued background job must be removed again, the runner processing is
+        # covered by test_backgroundjob.py
+        jobs = [job for job in BackgroundJob.objects.all() if job.payload.get('fileId') == model_obj.id]
+        self.assertEqual(len(jobs), 1)
+        jobs[0].delete()
+
     def test_helper_get_file(self):
         model_obj = self.create_test_object()
 

ci/apiv2/test_permissions.py

@@ -1686,11 +1686,27 @@ def test_api_token_recount_file_lines_helper_allowed_with_file_update_scope(self
             payload={'fileId': file_obj.id},
         )
         self.assertEqual(response.status_code, 200, response.text)
-        # The recounted file comes back as a resource object under data
-        data = response.json()['data']
-        self.assertEqual(data['type'], 'file')
-        self.assertEqual(data['id'], file_obj.id)
-        self.assertEqual(data['attributes']['filename'], file_obj.filename)
+        # The recounted file is returned as a resource object under data
+        body = response.json()
+        self.assertEqual(body['data']['type'], 'file')
+        self.assertEqual(body['data']['id'], file_obj.id)
+
+        # The recounting is enqueued as a background job, remove it again with the admin
+        # session as the API token scope does not allow background job deletion.
+        connector = File.objects.get_conn()
+        connector.authenticate()
+        response = requests.get(
+            connector._api_endpoint + '/ui/backgroundJobs',
+            headers=connector._headers,
+        )
+        self.assertEqual(response.status_code, 200, response.text)
+        jobs = [job for job in response.json()['data'] if job['attributes']['payload'].get('fileId') == file_obj.id]
+        self.assertEqual(len(jobs), 1)
+        response = requests.delete(
+            connector._api_endpoint + f"/ui/backgroundJobs/{jobs[0]['id']}",
+            headers=connector._headers,
+        )
+        self.assertEqual(response.status_code, 204, response.text)
 
     def test_api_token_abort_chunk_helper_allowed_with_chunk_update_scope(self):
         """abortChunk helper succeeds with permChunkUpdate.

ci/apiv2/utils.py

@@ -44,6 +44,10 @@ def delete(obj):
         pass  # we override the delete function for the tests as tokens cannot be deleted, but the teardown always calls delete after a test
 
 
+class BackgroundJob(Model, uri="/ui/backgroundJobs"):
+    pass  # background jobs are read-only via API, but queued or finished jobs can be deleted
+
+
 def get_test_config():
     load_order = (str(Path(__file__).parent.joinpath('{name}-defaults{suffix}')),) \
                  + confidence.DEFAULT_LOAD_ORDER
@@ -383,6 +387,7 @@ def find_stale_test_objects():
     test_objs.extend(Pretask.objects.all())
     test_objs.extend(Hashlist.objects.all())
     test_objs.extend(File.objects.all())
+    test_objs.extend(BackgroundJob.objects.all())
     test_objs.extend(User.objects.filter(id__gt=1))
     test_objs.extend(GlobalPermissionGroup.objects.filter(id__gt=1))
     test_objs.extend(Cracker.objects.filter(id__gt=1))

ci/phpunit/dba/AbstractModelFactoryTest.php

@@ -1173,13 +1173,14 @@ public function testMassSingleUpdateNoEffect(): void {
    */
   public function testMassUpdateSuccess(): void {
     $testId = uniqid();
-    $this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, 'hashtype1' . $testId, 1, 0));
+    $this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, 'hashtype1' . $testId, 0, 0));
     $this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, 'hashtype2' . $testId, 125, 0));
     $this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, 'hashtype3' . $testId, 72, 0));
 
     $uS = new UpdateSet(HashType::IS_SALTED, 1);
     $qF = new LikeFilter(HashType::DESCRIPTION, "%" . $testId);
-    Factory::getHashTypeFactory()->massUpdate([Factory::UPDATE => $uS, Factory::FILTER => $qF]);
+    $stmt = Factory::getHashTypeFactory()->massUpdate([Factory::UPDATE => $uS, Factory::FILTER => $qF]);
+    $this->assertSame(3, $stmt->rowCount());
 
     $sum = Factory::getHashTypeFactory()->sumFilter([Factory::FILTER => $qF], HashType::IS_SALTED);
     $this->assertEquals(3, $sum);
@@ -1198,7 +1199,8 @@ public function testMassUpdateNoEffect(): void {
 
     $uS = new UpdateSet(HashType::IS_SALTED, 1);
     $qF = new LikeFilter(HashType::DESCRIPTION, "%aaaa" . $testId);
-    Factory::getHashTypeFactory()->massUpdate([Factory::UPDATE => $uS, Factory::FILTER => $qF]);
+    $stmt = Factory::getHashTypeFactory()->massUpdate([Factory::UPDATE => $uS, Factory::FILTER => $qF]);
+    $this->assertSame(0, $stmt->rowCount());
 
     $qF = new LikeFilter(HashType::DESCRIPTION, "%" . $testId);
     $sum = Factory::getHashTypeFactory()->sumFilter([Factory::FILTER => $qF], HashType::IS_SALTED);

ci/phpunit/dba/UpdateSetTest.php

@@ -70,12 +70,12 @@ public function testMassUpdateSingleSet(): void {
     $scope = new LikeFilter(HashType::DESCRIPTION, '%' . $testId);
     $update = new UpdateSet(HashType::IS_SALTED, 99);
 
-    $result = Factory::getHashTypeFactory()->massUpdate([
+    $stmt = Factory::getHashTypeFactory()->massUpdate([
       Factory::UPDATE => $update,
       Factory::FILTER => $scope,
     ]);
 
-    $this->assertTrue($result);
+    $this->assertSame(3, $stmt->rowCount());
 
     $results = Factory::getHashTypeFactory()->filter([Factory::FILTER => $scope]);
     $this->assertCount(3, $results);
@@ -101,12 +101,12 @@ public function testMassUpdateMultipleSets(): void {
       new UpdateSet(HashType::IS_SLOW_HASH, 88),
     ];
 
-    $result = Factory::getHashTypeFactory()->massUpdate([
+    $stmt = Factory::getHashTypeFactory()->massUpdate([
       Factory::UPDATE => $updates,
       Factory::FILTER => $scope,
     ]);
 
-    $this->assertTrue($result);
+    $this->assertSame(2, $stmt->rowCount());
 
     $results = Factory::getHashTypeFactory()->filter([Factory::FILTER => $scope]);
     $this->assertCount(2, $results);

ci/phpunit/inc/UtilTest.php

@@ -1193,4 +1193,58 @@ public function testBuildBackendBaseUrlRejectsInvalidScheme(): void {
     $this->expectException(Exception::class);
     Util::buildBackendBaseUrl();
   }
+
+  /**
+   * fileLineCount returns -1 if the file does not exist.
+   */
+  public function testFileLineCountFileNotFound(): void {
+    $this->assertEquals(-1, Util::fileLineCount('/tmp/hashtopolis_util_nonexistent_file'));
+  }
+
+  /**
+   * fileLineCount counts the lines of a plain file.
+   */
+  public function testFileLineCountPlainFile(): void {
+    $path = tempnam(sys_get_temp_dir(), 'hashtopolis_util_');
+    file_put_contents($path, "line1\nline2\nline3\n");
+    $this->assertEquals(3, Util::fileLineCount($path));
+    unlink($path);
+  }
+
+  /**
+   * fileLineCount counts the lines of the decompressed stream of a .gz file
+   * (it is assumed that the archive contains exactly one file).
+   */
+  public function testFileLineCountGzFile(): void {
+    $path = tempnam(sys_get_temp_dir(), 'hashtopolis_util_') . '.gz';
+    $plainPath = tempnam(sys_get_temp_dir(), 'hashtopolis_util_');
+    file_put_contents($plainPath, "line1\nline2\nline3\n");
+    $gz = gzopen($path, 'wb');
+    gzwrite($gz, file_get_contents($plainPath));
+    gzclose($gz);
+    unlink($plainPath);
+    $this->assertEquals(3, Util::fileLineCount($path));
+    unlink($path);
+  }
+
+  /**
+   * fileLineCount counts zero lines for an empty .gz file.
+   */
+  public function testFileLineCountGzFileEmpty(): void {
+    $path = tempnam(sys_get_temp_dir(), 'hashtopolis_util_') . '.gz';
+    $gz = gzopen($path, 'wb');
+    gzclose($gz);
+    $this->assertEquals(0, Util::fileLineCount($path));
+    unlink($path);
+  }
+
+  /**
+   * fileLineCount returns -1 for a corrupted .gz file which cannot be decompressed.
+   */
+  public function testFileLineCountGzFileCorrupted(): void {
+    $path = tempnam(sys_get_temp_dir(), 'hashtopolis_util_') . '.gz';
+    file_put_contents($path, "this is not gzip data\n");
+    $this->assertEquals(-1, Util::fileLineCount($path));
+    unlink($path);
+  }
 }