diff --git a/.devcontainer/docker-compose.mysql.yml b/.devcontainer/docker-compose.mysql.yml
index acf22971e..731037d18 100644
--- a/.devcontainer/docker-compose.mysql.yml
+++ b/.devcontainer/docker-compose.mysql.yml
@@ -18,6 +18,7 @@ services:
# origin instead of a wildcard, so the dev frontend on :4200 has to be named here.
HASHTOPOLIS_BACKEND_URL: http://localhost:8080/api/v2
HASHTOPOLIS_FRONTEND_URLS: http://localhost:4200,http://127.0.0.1:4200
+ HASHTOPOLIS_CRON_ENABLE: 0
depends_on:
- hashtopolis-db-dev
ports:
diff --git a/.devcontainer/docker-compose.postgres.yml b/.devcontainer/docker-compose.postgres.yml
index dcd57fd6a..5292099b4 100644
--- a/.devcontainer/docker-compose.postgres.yml
+++ b/.devcontainer/docker-compose.postgres.yml
@@ -18,6 +18,7 @@ services:
# origin instead of a wildcard, so the dev frontend on :4200 has to be named here.
HASHTOPOLIS_BACKEND_URL: http://localhost:8080/api/v2
HASHTOPOLIS_FRONTEND_URLS: http://localhost:4200,http://127.0.0.1:4200
+ HASHTOPOLIS_CRON_ENABLE: 0
depends_on:
- hashtopolis-db-dev
ports:
diff --git a/.github/docker-compose.mysql.yml b/.github/docker-compose.mysql.yml
index 33dcac553..6761fa65b 100644
--- a/.github/docker-compose.mysql.yml
+++ b/.github/docker-compose.mysql.yml
@@ -16,6 +16,9 @@ services:
HASHTOPOLIS_DB_DATABASE: hashtopolis
HASHTOPOLIS_APIV2_ENABLE: 1
HASHTOPOLIS_BACKEND_URL: http://localhost:80/api/v2
+ # The background job runner is invoked explicitly by the test suite for deterministic
+ # results, therefore the entrypoint scheduler loop is disabled in the test container.
+ HASHTOPOLIS_CRON_ENABLE: 0
depends_on:
- hashtopolis-db-dev
ports:
diff --git a/.github/docker-compose.postgres.yml b/.github/docker-compose.postgres.yml
index b2ac28bcc..2894720c8 100644
--- a/.github/docker-compose.postgres.yml
+++ b/.github/docker-compose.postgres.yml
@@ -16,6 +16,9 @@ services:
HASHTOPOLIS_DB_DATABASE: hashtopolis
HASHTOPOLIS_APIV2_ENABLE: 1
HASHTOPOLIS_BACKEND_URL: http://localhost:80/api/v2
+ # The background job runner is invoked explicitly by the test suite for deterministic
+ # results, therefore the entrypoint scheduler loop is disabled in the test container.
+ HASHTOPOLIS_CRON_ENABLE: 0
depends_on:
- hashtopolis-db-dev
ports:
diff --git a/ci/apiv2/test_backgroundjob.py b/ci/apiv2/test_backgroundjob.py
new file mode 100644
index 000000000..f2c3888f5
--- /dev/null
+++ b/ci/apiv2/test_backgroundjob.py
@@ -0,0 +1,119 @@
+import subprocess
+
+import requests
+
+from hashtopolis import File, Helper
+from utils import BaseTest, BackgroundJob, create_restricted_user, get_hashtopolis_uri
+
+CRON_RUNNER = '/var/www/html/src/inc/cron.php'
+
+
+def run_background_job_runner():
+ result = subprocess.run(
+ ['php', '-d', 'xdebug.mode=off', '-f', CRON_RUNNER],
+ capture_output=True,
+ text=True,
+ )
+ assert result.returncode == 0, f"Background job runner failed: {result.stderr}"
+
+
+def get_jobs_for_file(file_id):
+ return [job for job in BackgroundJob.objects.all() if job.payload.get('fileId') == file_id]
+
+
+def get_admin_connector():
+ connector = BackgroundJob.objects.get_conn()
+ connector.authenticate()
+ return connector
+
+
+def enqueue_recount_job(file_obj):
+ helper = Helper()
+ helper.recount_file_lines(file=file_obj)
+
+
+class BackgroundJobTest(BaseTest):
+ model_class = BackgroundJob
+
+ def create_test_object(self, *nargs, delete=True, **kwargs):
+ # Background jobs cannot be created directly via the API, they get enqueued by
+ # other operations. The recount file lines helper is the reference producer.
+ file_obj = self.create_file()
+ enqueue_recount_job(file_obj)
+ jobs = get_jobs_for_file(file_obj.id)
+ assert len(jobs) == 1
+ if delete:
+ self.delete_after_test(jobs[0])
+ return jobs[0]
+
+ def test_recount_file_job_is_enqueued_as_pending(self):
+ model_obj = self.create_test_object()
+
+ self.assertEqual(model_obj.jobType, 'recount_file')
+ self.assertEqual(model_obj.status, 0)
+ self.assertIn('fileId', model_obj.payload)
+ self.assertIsNotNone(model_obj.userId)
+ self.assertIsNotNone(model_obj.createdAt)
+ self.assertIsNone(model_obj.startedAt)
+ self.assertIsNone(model_obj.finishedAt)
+
+ def test_recount_file_job_is_processed_by_runner(self):
+ model_obj = self.create_test_object()
+ file_id = model_obj.payload['fileId']
+
+ run_background_job_runner()
+
+ job = self.model_class.objects.get(pk=model_obj.id)
+ self.assertEqual(job.status, 2)
+ self.assertEqual(job.exitCode, 0)
+ self.assertEqual(job.resultMessage, 'Recounted 3 lines.')
+ self.assertIsNotNone(job.startedAt)
+ self.assertIsNotNone(job.finishedAt)
+
+ file = File.objects.get(pk=file_id)
+ self.assertEqual(file.lineCount, 3)
+
+ def test_delete_pending_job(self):
+ model_obj = self.create_test_object(delete=False)
+
+ # A pending (queued) job can be deleted, which cancels it
+ model_obj.delete()
+
+ with self.assertRaises(self.model_class.DoesNotExist):
+ _ = self.model_class.objects.get(pk=model_obj.id)
+
+ def test_delete_processed_job(self):
+ model_obj = self.create_test_object(delete=False)
+
+ run_background_job_runner()
+ model_obj.delete()
+
+ with self.assertRaises(self.model_class.DoesNotExist):
+ _ = self.model_class.objects.get(pk=model_obj.id)
+
+ def test_create_via_api_not_allowed(self):
+ connector = get_admin_connector()
+ response = requests.post(
+ connector._api_endpoint + '/ui/backgroundJobs',
+ headers=connector._headers,
+ data='{"data": {"type": "backgroundJob", "attributes": {}}}',
+ )
+ self.assertEqual(response.status_code, 405, response.text)
+
+ def test_restricted_user_cannot_access_without_server_config_access(self):
+ self.create_test_object()
+
+ auth = create_restricted_user(self, {'permFileRead': True})
+ response = requests.post(
+ get_hashtopolis_uri() + '/api/v2/auth/token',
+ auth=auth,
+ )
+ self.assertIn(response.status_code, [200, 201], response.text)
+ token = response.json()['token']
+
+ connector = get_admin_connector()
+ response = requests.get(
+ connector._api_endpoint + '/ui/backgroundJobs',
+ headers={'Authorization': f'Bearer {token}'},
+ )
+ self.assertEqual(response.status_code, 403, response.text)
diff --git a/ci/apiv2/test_file.py b/ci/apiv2/test_file.py
index 9678b717c..2655154f8 100644
--- a/ci/apiv2/test_file.py
+++ b/ci/apiv2/test_file.py
@@ -1,7 +1,7 @@
import pytest
from hashtopolis import File, Helper
-from utils import BaseTest
+from utils import BaseTest, BackgroundJob
class FileTest(BaseTest):
@@ -32,9 +32,8 @@ def test_create_binary(self):
self._test_create(model_obj)
def test_recount_wordlist(self):
- # Note: After the object creation, the line count is already updated, but afterward it is immutable on the API.
- # There the test just check that the API function is callable and returns the file, but the count is
- # already the same beforehand.
+ # Note: The recounting is enqueued as background job, but it was already done on the object creation,
+ # the file is returned with its immutable line count.
model_obj = self.create_test_object()
helper = Helper()
@@ -42,6 +41,12 @@ def test_recount_wordlist(self):
self.assertEqual(file.lineCount, 3)
+ # The enqueued background job must be removed again, the runner processing is
+ # covered by test_backgroundjob.py
+ jobs = [job for job in BackgroundJob.objects.all() if job.payload.get('fileId') == model_obj.id]
+ self.assertEqual(len(jobs), 1)
+ jobs[0].delete()
+
def test_helper_get_file(self):
model_obj = self.create_test_object()
diff --git a/ci/apiv2/test_permissions.py b/ci/apiv2/test_permissions.py
index bb1c28b6f..dc9bd0b80 100644
--- a/ci/apiv2/test_permissions.py
+++ b/ci/apiv2/test_permissions.py
@@ -1686,11 +1686,27 @@ def test_api_token_recount_file_lines_helper_allowed_with_file_update_scope(self
payload={'fileId': file_obj.id},
)
self.assertEqual(response.status_code, 200, response.text)
- # The recounted file comes back as a resource object under data
- data = response.json()['data']
- self.assertEqual(data['type'], 'file')
- self.assertEqual(data['id'], file_obj.id)
- self.assertEqual(data['attributes']['filename'], file_obj.filename)
+ # The recounted file is returned as a resource object under data
+ body = response.json()
+ self.assertEqual(body['data']['type'], 'file')
+ self.assertEqual(body['data']['id'], file_obj.id)
+
+ # The recounting is enqueued as a background job, remove it again with the admin
+ # session as the API token scope does not allow background job deletion.
+ connector = File.objects.get_conn()
+ connector.authenticate()
+ response = requests.get(
+ connector._api_endpoint + '/ui/backgroundJobs',
+ headers=connector._headers,
+ )
+ self.assertEqual(response.status_code, 200, response.text)
+ jobs = [job for job in response.json()['data'] if job['attributes']['payload'].get('fileId') == file_obj.id]
+ self.assertEqual(len(jobs), 1)
+ response = requests.delete(
+ connector._api_endpoint + f"/ui/backgroundJobs/{jobs[0]['id']}",
+ headers=connector._headers,
+ )
+ self.assertEqual(response.status_code, 204, response.text)
def test_api_token_abort_chunk_helper_allowed_with_chunk_update_scope(self):
"""abortChunk helper succeeds with permChunkUpdate.
diff --git a/ci/apiv2/utils.py b/ci/apiv2/utils.py
index 4e399aeab..760ad8977 100644
--- a/ci/apiv2/utils.py
+++ b/ci/apiv2/utils.py
@@ -44,6 +44,10 @@ def delete(obj):
pass # we override the delete function for the tests as tokens cannot be deleted, but the teardown always calls delete after a test
+class BackgroundJob(Model, uri="/ui/backgroundJobs"):
+ pass # background jobs are read-only via API, but queued or finished jobs can be deleted
+
+
def get_test_config():
load_order = (str(Path(__file__).parent.joinpath('{name}-defaults{suffix}')),) \
+ confidence.DEFAULT_LOAD_ORDER
@@ -383,6 +387,7 @@ def find_stale_test_objects():
test_objs.extend(Pretask.objects.all())
test_objs.extend(Hashlist.objects.all())
test_objs.extend(File.objects.all())
+ test_objs.extend(BackgroundJob.objects.all())
test_objs.extend(User.objects.filter(id__gt=1))
test_objs.extend(GlobalPermissionGroup.objects.filter(id__gt=1))
test_objs.extend(Cracker.objects.filter(id__gt=1))
diff --git a/ci/phpunit/dba/AbstractModelFactoryTest.php b/ci/phpunit/dba/AbstractModelFactoryTest.php
index 9f5acce76..a488835bb 100644
--- a/ci/phpunit/dba/AbstractModelFactoryTest.php
+++ b/ci/phpunit/dba/AbstractModelFactoryTest.php
@@ -1173,13 +1173,14 @@ public function testMassSingleUpdateNoEffect(): void {
*/
public function testMassUpdateSuccess(): void {
$testId = uniqid();
- $this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, 'hashtype1' . $testId, 1, 0));
+ $this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, 'hashtype1' . $testId, 0, 0));
$this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, 'hashtype2' . $testId, 125, 0));
$this->createDatabaseObject(Factory::getHashTypeFactory(), new HashType(null, 'hashtype3' . $testId, 72, 0));
$uS = new UpdateSet(HashType::IS_SALTED, 1);
$qF = new LikeFilter(HashType::DESCRIPTION, "%" . $testId);
- Factory::getHashTypeFactory()->massUpdate([Factory::UPDATE => $uS, Factory::FILTER => $qF]);
+ $stmt = Factory::getHashTypeFactory()->massUpdate([Factory::UPDATE => $uS, Factory::FILTER => $qF]);
+ $this->assertSame(3, $stmt->rowCount());
$sum = Factory::getHashTypeFactory()->sumFilter([Factory::FILTER => $qF], HashType::IS_SALTED);
$this->assertEquals(3, $sum);
@@ -1198,7 +1199,8 @@ public function testMassUpdateNoEffect(): void {
$uS = new UpdateSet(HashType::IS_SALTED, 1);
$qF = new LikeFilter(HashType::DESCRIPTION, "%aaaa" . $testId);
- Factory::getHashTypeFactory()->massUpdate([Factory::UPDATE => $uS, Factory::FILTER => $qF]);
+ $stmt = Factory::getHashTypeFactory()->massUpdate([Factory::UPDATE => $uS, Factory::FILTER => $qF]);
+ $this->assertSame(0, $stmt->rowCount());
$qF = new LikeFilter(HashType::DESCRIPTION, "%" . $testId);
$sum = Factory::getHashTypeFactory()->sumFilter([Factory::FILTER => $qF], HashType::IS_SALTED);
diff --git a/ci/phpunit/dba/UpdateSetTest.php b/ci/phpunit/dba/UpdateSetTest.php
index e8a163373..26ed08e66 100644
--- a/ci/phpunit/dba/UpdateSetTest.php
+++ b/ci/phpunit/dba/UpdateSetTest.php
@@ -70,12 +70,12 @@ public function testMassUpdateSingleSet(): void {
$scope = new LikeFilter(HashType::DESCRIPTION, '%' . $testId);
$update = new UpdateSet(HashType::IS_SALTED, 99);
- $result = Factory::getHashTypeFactory()->massUpdate([
+ $stmt = Factory::getHashTypeFactory()->massUpdate([
Factory::UPDATE => $update,
Factory::FILTER => $scope,
]);
- $this->assertTrue($result);
+ $this->assertSame(3, $stmt->rowCount());
$results = Factory::getHashTypeFactory()->filter([Factory::FILTER => $scope]);
$this->assertCount(3, $results);
@@ -101,12 +101,12 @@ public function testMassUpdateMultipleSets(): void {
new UpdateSet(HashType::IS_SLOW_HASH, 88),
];
- $result = Factory::getHashTypeFactory()->massUpdate([
+ $stmt = Factory::getHashTypeFactory()->massUpdate([
Factory::UPDATE => $updates,
Factory::FILTER => $scope,
]);
- $this->assertTrue($result);
+ $this->assertSame(2, $stmt->rowCount());
$results = Factory::getHashTypeFactory()->filter([Factory::FILTER => $scope]);
$this->assertCount(2, $results);
diff --git a/ci/phpunit/inc/UtilTest.php b/ci/phpunit/inc/UtilTest.php
index 2b756345c..e3405faad 100644
--- a/ci/phpunit/inc/UtilTest.php
+++ b/ci/phpunit/inc/UtilTest.php
@@ -1193,4 +1193,58 @@ public function testBuildBackendBaseUrlRejectsInvalidScheme(): void {
$this->expectException(Exception::class);
Util::buildBackendBaseUrl();
}
+
+ /**
+ * fileLineCount returns -1 if the file does not exist.
+ */
+ public function testFileLineCountFileNotFound(): void {
+ $this->assertEquals(-1, Util::fileLineCount('/tmp/hashtopolis_util_nonexistent_file'));
+ }
+
+ /**
+ * fileLineCount counts the lines of a plain file.
+ */
+ public function testFileLineCountPlainFile(): void {
+ $path = tempnam(sys_get_temp_dir(), 'hashtopolis_util_');
+ file_put_contents($path, "line1\nline2\nline3\n");
+ $this->assertEquals(3, Util::fileLineCount($path));
+ unlink($path);
+ }
+
+ /**
+ * fileLineCount counts the lines of the decompressed stream of a .gz file
+ * (it is assumed that the archive contains exactly one file).
+ */
+ public function testFileLineCountGzFile(): void {
+ $path = tempnam(sys_get_temp_dir(), 'hashtopolis_util_') . '.gz';
+ $plainPath = tempnam(sys_get_temp_dir(), 'hashtopolis_util_');
+ file_put_contents($plainPath, "line1\nline2\nline3\n");
+ $gz = gzopen($path, 'wb');
+ gzwrite($gz, file_get_contents($plainPath));
+ gzclose($gz);
+ unlink($plainPath);
+ $this->assertEquals(3, Util::fileLineCount($path));
+ unlink($path);
+ }
+
+ /**
+ * fileLineCount counts zero lines for an empty .gz file.
+ */
+ public function testFileLineCountGzFileEmpty(): void {
+ $path = tempnam(sys_get_temp_dir(), 'hashtopolis_util_') . '.gz';
+ $gz = gzopen($path, 'wb');
+ gzclose($gz);
+ $this->assertEquals(0, Util::fileLineCount($path));
+ unlink($path);
+ }
+
+ /**
+ * fileLineCount returns -1 for a corrupted .gz file which cannot be decompressed.
+ */
+ public function testFileLineCountGzFileCorrupted(): void {
+ $path = tempnam(sys_get_temp_dir(), 'hashtopolis_util_') . '.gz';
+ file_put_contents($path, "this is not gzip data\n");
+ $this->assertEquals(-1, Util::fileLineCount($path));
+ unlink($path);
+ }
}
diff --git a/ci/phpunit/inc/jobs/BackgroundJobPayloadValidatorTest.php b/ci/phpunit/inc/jobs/BackgroundJobPayloadValidatorTest.php
new file mode 100644
index 000000000..f1393c0a8
--- /dev/null
+++ b/ci/phpunit/inc/jobs/BackgroundJobPayloadValidatorTest.php
@@ -0,0 +1,114 @@
+ 123, 'note' => 'abc'],
+ ['fileId' => new JobPayloadField(JobPayloadField::TYPE_INT)]
+ );
+ $this->expectNotToPerformAssertions();
+ }
+
+ /**
+ * @throws HTException
+ */
+ public function testMissingRequiredKeyThrows(): void {
+ $this->expectException(HTException::class);
+ $this->expectExceptionMessage("Missing required payload key 'fileId'!");
+ BackgroundJobPayloadValidator::validate([], ['fileId' => new JobPayloadField(JobPayloadField::TYPE_INT)]);
+ }
+
+ /**
+ * @throws HTException
+ */
+ public function testMissingOptionalKeyPasses(): void {
+ BackgroundJobPayloadValidator::validate(
+ ['fileId' => 123],
+ [
+ 'fileId' => new JobPayloadField(JobPayloadField::TYPE_INT),
+ 'note' => new JobPayloadField(JobPayloadField::TYPE_STRING, false),
+ ]
+ );
+ $this->expectNotToPerformAssertions();
+ }
+
+ /**
+ * @throws HTException
+ */
+ public function testNullValueThrowsEvenWhenOptional(): void {
+ $this->expectException(HTException::class);
+ $this->expectExceptionMessage("Invalid type for payload key 'note', expected string!");
+ BackgroundJobPayloadValidator::validate(
+ ['note' => null],
+ ['note' => new JobPayloadField(JobPayloadField::TYPE_STRING, false)]
+ );
+ }
+
+ /**
+ * @throws HTException
+ */
+ public function testUndeclaredKeysAreIgnored(): void {
+ BackgroundJobPayloadValidator::validate(
+ ['fileId' => 123, 'unknown' => 'whatever'],
+ ['fileId' => new JobPayloadField(JobPayloadField::TYPE_INT)]
+ );
+ $this->expectNotToPerformAssertions();
+ }
+
+ /**
+ * @param string $type one of the JobPayloadField type constants
+ * @param mixed $value value matching the type
+ * @throws HTException
+ */
+ #[DataProvider('provideTypes')]
+ public function testMatchingTypesPass(string $type, mixed $value): void {
+ BackgroundJobPayloadValidator::validate(['key' => $value], ['key' => new JobPayloadField($type)]);
+ $this->expectNotToPerformAssertions();
+ }
+
+ /**
+ * @param string $type one of the JobPayloadField type constants
+ * @param mixed $value value violating the type
+ * @throws HTException
+ */
+ #[DataProvider('provideTypeMismatches')]
+ public function testTypeMismatchThrows(string $type, mixed $value): void {
+ $this->expectException(HTException::class);
+ $this->expectExceptionMessage("Invalid type for payload key 'key', expected $type!");
+ BackgroundJobPayloadValidator::validate(['key' => $value], ['key' => new JobPayloadField($type)]);
+ }
+}
diff --git a/ci/phpunit/inc/utils/BackgroundJobUtilsTest.php b/ci/phpunit/inc/utils/BackgroundJobUtilsTest.php
new file mode 100644
index 000000000..1959a640c
--- /dev/null
+++ b/ci/phpunit/inc/utils/BackgroundJobUtilsTest.php
@@ -0,0 +1,229 @@
+testFilename = 'phpunit_bgjob_' . uniqid() . '.txt';
+ }
+
+ #[Override]
+ protected function tearDown(): void {
+ $filePath = self::getFilesDir() . '/' . $this->testFilename;
+ if (is_file($filePath)) {
+ unlink($filePath);
+ }
+ parent::tearDown();
+ }
+
+ private static function getFilesDir(): string {
+ return Factory::getStoredValueFactory()->get(DDirectories::FILES)->getVal();
+ }
+
+ /**
+ * @throws Exception
+ */
+ private function createPhysicalFile(string $content): void {
+ file_put_contents(self::getFilesDir() . '/' . $this->testFilename, $content);
+ }
+
+ /**
+ * @throws Exception
+ */
+ private function enqueueRecountFileJob(int $fileId, ?User $user = null): BackgroundJob {
+ $job = BackgroundJobUtils::enqueue(DBackgroundJobType::RECOUNT_FILE, ['fileId' => $fileId], $user);
+ $this->registerDatabaseObject(Factory::getBackgroundJobFactory(), $job);
+ return $job;
+ }
+
+ /**
+ * @throws HTException
+ */
+ public function testEnqueueCreatesPendingJobWithUserAndPayload(): void {
+ $user = $this->createUser("phpunitBgJob");
+ $job = $this->enqueueRecountFileJob(123, $user);
+
+ $this->assertSame(DBackgroundJobStatus::PENDING, $job->getStatus());
+ $this->assertSame($user->getId(), $job->getUserId());
+ $this->assertGreaterThan(0, $job->getId());
+ $this->assertNotNull($job->getCreatedAt());
+ $this->assertNull($job->getStartedAt());
+
+ // payload is stored as native JSON and round-trips through the DBA layer
+ $job = Factory::getBackgroundJobFactory()->get($job->getId());
+ $this->assertNotNull($job);
+ $this->assertSame(['fileId' => 123], json_decode($job->getPayload(), true, 512, JSON_THROW_ON_ERROR));
+ }
+
+ /**
+ * @throws HTException
+ */
+ public function testEnqueueWithoutUserRecordsSystemAsTrigger(): void {
+ $job = $this->enqueueRecountFileJob(123);
+ $this->assertNull($job->getUserId());
+ }
+
+ /**
+ * @throws HTException
+ */
+ public function testEnqueueUnknownJobTypeThrows(): void {
+ $this->expectException(HTException::class);
+ BackgroundJobUtils::enqueue('unknown_job_type', []);
+ }
+
+ /**
+ * @throws HTException
+ */
+ public function testEnqueueMissingRequiredPayloadKeyThrows(): void {
+ $this->expectException(HTException::class);
+ $this->expectExceptionMessage("Missing required payload key 'fileId'!");
+ BackgroundJobUtils::enqueue(DBackgroundJobType::RECOUNT_FILE, []);
+ }
+
+ /**
+ * @throws HTException
+ */
+ public function testEnqueueInvalidPayloadTypeThrows(): void {
+ $this->expectException(HTException::class);
+ $this->expectExceptionMessage("Invalid type for payload key 'fileId', expected int!");
+ BackgroundJobUtils::enqueue(DBackgroundJobType::RECOUNT_FILE, ['fileId' => '123']);
+ }
+
+ /**
+ * @throws Exception
+ */
+ public function testDeleteRejectsJobClaimedAfterFetch(): void {
+ $job = $this->enqueueRecountFileJob(123);
+ $factory = Factory::getBackgroundJobFactory();
+ $stmt = $factory->massUpdate([
+ Factory::UPDATE => new UpdateSet(BackgroundJob::STATUS, DBackgroundJobStatus::RUNNING),
+ Factory::FILTER => [
+ new QueryFilter(BackgroundJob::BACKGROUND_JOB_ID, $job->getId(), "="),
+ new QueryFilter(BackgroundJob::STATUS, DBackgroundJobStatus::PENDING, "="),
+ ],
+ ]);
+ $this->assertSame(1, $stmt->rowCount());
+
+ try {
+ BackgroundJobUtils::deleteJob($job);
+ $this->fail("Deleting a running background job should fail");
+ }
+ catch (HttpConflict) {
+ $storedJob = $factory->get($job->getId());
+ $this->assertNotNull($storedJob);
+ $this->assertSame(DBackgroundJobStatus::RUNNING, $storedJob->getStatus());
+ }
+ }
+
+ /**
+ * @throws Exception
+ */
+ public function testRunnerProcessesRecountFileJob(): void {
+ $user = $this->createUser("phpunitBgJob");
+ $file = $this->createFile($this->createAccessGroup("phpunitBgJob"), 0, $this->testFilename);
+ $this->createPhysicalFile("one\ntwo\nthree\n");
+
+ $job = $this->enqueueRecountFileJob($file->getId(), $user);
+ BackgroundJobRunner::run();
+
+ $job = Factory::getBackgroundJobFactory()->get($job->getId());
+ $this->assertSame(DBackgroundJobStatus::DONE, $job->getStatus());
+ $this->assertSame(0, $job->getExitCode());
+ $this->assertSame("Recounted 3 lines.", $job->getResultMessage());
+ $this->assertNotNull($job->getStartedAt());
+ $this->assertNotNull($job->getFinishedAt());
+
+ $file = Factory::getFileFactory()->get($file->getId());
+ $this->assertSame(3, $file->getLineCount());
+ }
+
+ /**
+ * @throws Exception
+ */
+ public function testRunnerFailsJobOnMissingFile(): void {
+ $job = $this->enqueueRecountFileJob(999999999);
+ BackgroundJobRunner::run();
+
+ $job = Factory::getBackgroundJobFactory()->get($job->getId());
+ $this->assertSame(DBackgroundJobStatus::FAILED, $job->getStatus());
+ $this->assertSame(-1, $job->getExitCode());
+ $this->assertSame("No such file!", $job->getResultMessage());
+ $this->assertNotNull($job->getFinishedAt());
+ }
+
+ /**
+ * @throws Exception
+ */
+ public function testRunnerMarksStaleRunningJobAsFailed(): void {
+ $job = $this->createDatabaseObject(
+ Factory::getBackgroundJobFactory(),
+ new BackgroundJob(null, DBackgroundJobType::RECOUNT_FILE, '{"fileId": 1}', DBackgroundJobStatus::RUNNING, null, time(), time() - 8000, null, null, null)
+ );
+ BackgroundJobRunner::run();
+
+ $job = Factory::getBackgroundJobFactory()->get($job->getId());
+ $this->assertSame(DBackgroundJobStatus::FAILED, $job->getStatus());
+ $this->assertSame(-1, $job->getExitCode());
+ $this->assertStringContainsString("timed out", $job->getResultMessage());
+ }
+
+ /**
+ * @throws Exception
+ */
+ public function testRunnerFailsJobWithPayloadViolatingDefinition(): void {
+ // bypassing enqueue() on purpose to simulate a legacy or manually written job row
+ $job = $this->createDatabaseObject(
+ Factory::getBackgroundJobFactory(),
+ new BackgroundJob(null, DBackgroundJobType::RECOUNT_FILE, '{"fileId": "abc"}', DBackgroundJobStatus::PENDING, null, time(), null, null, null, null)
+ );
+ BackgroundJobRunner::run();
+
+ $job = Factory::getBackgroundJobFactory()->get($job->getId());
+ $this->assertSame(DBackgroundJobStatus::FAILED, $job->getStatus());
+ $this->assertSame(255, $job->getExitCode());
+ $this->assertSame("Invalid type for payload key 'fileId', expected int!", $job->getResultMessage());
+ $this->assertNotNull($job->getFinishedAt());
+ }
+
+ /**
+ * @throws Exception
+ */
+ public function testRunnerProcessesAllPendingJobsAfterEachOther(): void {
+ $file = $this->createFile($this->createAccessGroup("phpunitBgJob"), 0, $this->testFilename);
+ $this->createPhysicalFile("one\n");
+
+ $first = $this->enqueueRecountFileJob(999999998);
+ $second = $this->enqueueRecountFileJob($file->getId());
+ BackgroundJobRunner::run();
+
+ $first = Factory::getBackgroundJobFactory()->get($first->getId());
+ $second = Factory::getBackgroundJobFactory()->get($second->getId());
+ $this->assertSame(DBackgroundJobStatus::FAILED, $first->getStatus());
+ $this->assertSame(DBackgroundJobStatus::DONE, $second->getStatus());
+ // FIFO: the first enqueued job must have been started before the second one
+ $this->assertLessThanOrEqual($second->getStartedAt(), $first->getStartedAt());
+ }
+}
diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
index d6f7a8858..374b6a369 100755
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -116,7 +116,27 @@ echo ""
echo " Hashtopolis is now ready to use!"
echo " *\0/*"
echo ""
+
echo ""
+# Schedule the background job runner. It is called once a minute, overlapping runs
+# are prevented by the runner itself using a lock file. A cron daemon cannot be used
+# here as the container runs with an unprivileged user (vscode/www-data), so a simple
+# scheduler loop is started in the background instead. The loop inherits the container
+# environment, which a cron daemon would not. Set HASHTOPOLIS_CRON_ENABLE=0 to disable
+# it, for example to run the runner with an external cron or for deterministic tests.
+if [[ -z "${HASHTOPOLIS_CRON_ENABLE+x}" ]]; then
+ HASHTOPOLIS_CRON_ENABLE="1"
+fi
+if [[ "$HASHTOPOLIS_CRON_ENABLE" == "1" ]]; then
+ echo "Starting background job runner scheduler (every 60s)..."
+ (
+ while true; do
+ php -d xdebug.mode=off -f ${HASHTOPOLIS_DOCUMENT_ROOT}/inc/cron.php
+ sleep 60
+ done
+ ) &
+fi
+
docker-php-entrypoint apache2-foreground
diff --git a/openapi.json b/openapi.json
index ac20c851c..a84717e88 100644
--- a/openapi.json
+++ b/openapi.json
@@ -6040,10 +6040,10 @@
"operationId": "deleteApiTokensById"
}
},
- "/api/v2/ui/chunks": {
+ "/api/v2/ui/backgroundJobs": {
"get": {
"tags": [
- "Chunks"
+ "BackgroundJobs"
],
"responses": {
"400": {
@@ -6081,7 +6081,7 @@
"content": {
"application/vnd.api+json": {
"schema": {
- "$ref": "#/components/schemas/ChunkListResponse"
+ "$ref": "#/components/schemas/BackgroundJobListResponse"
}
}
}
@@ -6101,8 +6101,8 @@
"type": "string",
"format": "byte"
},
- "example": "eyJwcmltYXJ5Ijp7ImNodW5rSWQiOiAxMjN9fQ==",
- "description": "Pointer to paginate to retrieve the data after the object provided. Specify the `base64` encoded JSON string in a **uniquely identifiable** manner (e.g. object IDs), i.e. by using one (primary) or two (primary and secondary) fields that allow for **stable** sorting.\n \n\nFormat: `{\"primary\":{\"someField\": 123},\"secondary\":{\"someOtherOptionalField\": \"Foo\"}}`\n \n\nExample: `{\"primary\":{\"chunkId\": 123}}` -> `eyJwcmltYXJ5Ijp7ImNodW5rSWQiOiAxMjN9fQ==`"
+ "example": "eyJwcmltYXJ5Ijp7ImJhY2tncm91bmRKb2JJZCI6IDEyM319",
+ "description": "Pointer to paginate to retrieve the data after the object provided. Specify the `base64` encoded JSON string in a **uniquely identifiable** manner (e.g. object IDs), i.e. by using one (primary) or two (primary and secondary) fields that allow for **stable** sorting.\n \n\nFormat: `{\"primary\":{\"someField\": 123},\"secondary\":{\"someOtherOptionalField\": \"Foo\"}}`\n \n\nExample: `{\"primary\":{\"backgroundJobId\": 123}}` -> `eyJwcmltYXJ5Ijp7ImJhY2tncm91bmRKb2JJZCI6IDEyM319`"
},
{
"name": "page[before]",
@@ -6111,8 +6111,8 @@
"type": "string",
"format": "byte"
},
- "example": "eyJwcmltYXJ5Ijp7ImNodW5rSWQiOiAxMjN9fQ==",
- "description": "Pointer to paginate to retrieve the data before the object provided. Specify the `base64` encoded JSON string in a **uniquely identifiable** manner (e.g. object IDs), i.e. by using one (primary) or two (primary and secondary) fields that allow for **stable** sorting.\n \n\nFormat: `{\"primary\":{\"someField\": 123},\"secondary\":{\"someOtherOptionalField\": \"Foo\"}}`\n \n\nExample: `{\"primary\":{\"chunkId\": 123}}` -> `eyJwcmltYXJ5Ijp7ImNodW5rSWQiOiAxMjN9fQ==`"
+ "example": "eyJwcmltYXJ5Ijp7ImJhY2tncm91bmRKb2JJZCI6IDEyM319",
+ "description": "Pointer to paginate to retrieve the data before the object provided. Specify the `base64` encoded JSON string in a **uniquely identifiable** manner (e.g. object IDs), i.e. by using one (primary) or two (primary and secondary) fields that allow for **stable** sorting.\n \n\nFormat: `{\"primary\":{\"someField\": 123},\"secondary\":{\"someOtherOptionalField\": \"Foo\"}}`\n \n\nExample: `{\"primary\":{\"backgroundJobId\": 123}}` -> `eyJwcmltYXJ5Ijp7ImJhY2tncm91bmRKb2JJZCI6IDEyM319`"
},
{
"name": "page[size]",
@@ -6135,9 +6135,9 @@
"type": "string"
}
},
- "description": "Filters results using a query. Every key is an attribute name optionally suffixed with a comparison operator, e.g. `filter[chunkId__gt]=200`.",
+ "description": "Filters results using a query. Every key is an attribute name optionally suffixed with a comparison operator, e.g. `filter[backgroundJobId__gt]=200`.",
"example": {
- "chunkId__gt": "200"
+ "backgroundJobId__gt": "200"
}
},
{
@@ -6150,26 +6150,93 @@
"items": {
"type": "string",
"enum": [
- "agent",
- "task"
+ "user"
]
}
},
- "description": "Relationships to include in the response, comma seperated. Possible options: agent, task",
+ "description": "Relationships to include in the response, comma seperated. Possible options: user",
"example": [
- "agent",
- "task"
+ "user"
]
}
],
- "summary": "List Chunks",
- "operationId": "getChunks"
+ "summary": "List BackgroundJobs",
+ "operationId": "getBackgroundJobs"
+ },
+ "delete": {
+ "tags": [
+ "BackgroundJobs"
+ ],
+ "responses": {
+ "400": {
+ "description": "Invalid request",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "401": {
+ "description": "Authentication failed",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "403": {
+ "description": "Permission denied",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "204": {
+ "description": "successfully deleted"
+ },
+ "404": {
+ "description": "Not Found",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ }
+ },
+ "security": [
+ {
+ "bearerAuth": []
+ }
+ ],
+ "description": "Delete BackgroundJobs",
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/vnd.api+json": {
+ "schema": {
+ "$ref": "#/components/schemas/BackgroundJobDeleteMultiple"
+ }
+ }
+ }
+ },
+ "parameters": [],
+ "summary": "Delete BackgroundJobs",
+ "operationId": "deleteBackgroundJobs"
}
},
- "/api/v2/ui/chunks/count": {
+ "/api/v2/ui/backgroundJobs/count": {
"get": {
"tags": [
- "Chunks"
+ "BackgroundJobs"
],
"responses": {
"400": {
@@ -6207,7 +6274,7 @@
"content": {
"application/vnd.api+json": {
"schema": {
- "$ref": "#/components/schemas/ChunkCountResponse"
+ "$ref": "#/components/schemas/BackgroundJobCountResponse"
}
}
}
@@ -6231,9 +6298,9 @@
"type": "string"
}
},
- "description": "Filters results using a query. Every key is an attribute name optionally suffixed with a comparison operator, e.g. `filter[chunkId__gt]=200`.",
+ "description": "Filters results using a query. Every key is an attribute name optionally suffixed with a comparison operator, e.g. `filter[backgroundJobId__gt]=200`.",
"example": {
- "chunkId__gt": "200"
+ "backgroundJobId__gt": "200"
}
},
{
@@ -6246,14 +6313,14 @@
"description": "Also report the number of accessible objects without any filter applied, as `meta.total_count`"
}
],
- "summary": "Count Chunks",
- "operationId": "getChunksCount"
+ "summary": "Count BackgroundJobs",
+ "operationId": "getBackgroundJobsCount"
}
},
- "/api/v2/ui/chunks/{id}/{relation}": {
+ "/api/v2/ui/backgroundJobs/{id}/{relation}": {
"get": {
"tags": [
- "Chunks"
+ "BackgroundJobs"
],
"responses": {
"400": {
@@ -6301,7 +6368,7 @@
"content": {
"application/vnd.api+json": {
"schema": {
- "$ref": "#/components/schemas/ChunkRelationTaskGetResponse"
+ "$ref": "#/components/schemas/BackgroundJobRelationUserGetResponse"
}
}
}
@@ -6333,14 +6400,14 @@
}
}
],
- "summary": "Get Chunks",
- "operationId": "getChunksByIdByRelation"
+ "summary": "Get BackgroundJobs",
+ "operationId": "getBackgroundJobsByIdByRelation"
}
},
- "/api/v2/ui/chunks/{id}/relationships/{relation}": {
+ "/api/v2/ui/backgroundJobs/{id}/relationships/{relation}": {
"get": {
"tags": [
- "Chunks"
+ "BackgroundJobs"
],
"responses": {
"400": {
@@ -6388,7 +6455,7 @@
"content": {
"application/vnd.api+json": {
"schema": {
- "$ref": "#/components/schemas/ChunkResponse"
+ "$ref": "#/components/schemas/BackgroundJobResponse"
}
}
}
@@ -6420,12 +6487,12 @@
}
}
],
- "summary": "Get Chunks",
- "operationId": "getChunksByIdRelationshipsByRelation"
+ "summary": "Get BackgroundJobs",
+ "operationId": "getBackgroundJobsByIdRelationshipsByRelation"
},
"patch": {
"tags": [
- "Chunks"
+ "BackgroundJobs"
],
"responses": {
"400": {
@@ -6493,7 +6560,7 @@
"content": {
"application/vnd.api+json": {
"schema": {
- "$ref": "#/components/schemas/ChunkRelationTask"
+ "$ref": "#/components/schemas/BackgroundJobRelationUser"
}
}
}
@@ -6516,14 +6583,14 @@
}
}
],
- "summary": "Update Chunks",
- "operationId": "patchChunksByIdRelationshipsByRelation"
+ "summary": "Update BackgroundJobs",
+ "operationId": "patchBackgroundJobsByIdRelationshipsByRelation"
}
},
- "/api/v2/ui/chunks/{id}": {
+ "/api/v2/ui/backgroundJobs/{id}": {
"get": {
"tags": [
- "Chunks"
+ "BackgroundJobs"
],
"responses": {
"400": {
@@ -6571,7 +6638,7 @@
"content": {
"application/vnd.api+json": {
"schema": {
- "$ref": "#/components/schemas/ChunkResponse"
+ "$ref": "#/components/schemas/BackgroundJobResponse"
}
}
}
@@ -6604,165 +6671,22 @@
"items": {
"type": "string",
"enum": [
- "agent",
- "task"
- ]
- }
- },
- "description": "Relationships to include in the response, comma seperated. Possible options: agent, task",
- "example": [
- "agent",
- "task"
- ]
- }
- ],
- "summary": "Get Chunks",
- "operationId": "getChunksById"
- }
- },
- "/api/v2/ui/configs": {
- "get": {
- "tags": [
- "Configs"
- ],
- "responses": {
- "400": {
- "description": "Invalid request",
- "content": {
- "application/problem+json": {
- "schema": {
- "$ref": "#/components/schemas/ErrorResponse"
- }
- }
- }
- },
- "401": {
- "description": "Authentication failed",
- "content": {
- "application/problem+json": {
- "schema": {
- "$ref": "#/components/schemas/ErrorResponse"
- }
- }
- }
- },
- "403": {
- "description": "Permission denied",
- "content": {
- "application/problem+json": {
- "schema": {
- "$ref": "#/components/schemas/ErrorResponse"
- }
- }
- }
- },
- "200": {
- "description": "successful operation",
- "content": {
- "application/vnd.api+json": {
- "schema": {
- "$ref": "#/components/schemas/ConfigListResponse"
- }
- }
- }
- }
- },
- "security": [
- {
- "bearerAuth": []
- }
- ],
- "description": "GET many request to retrieve multiple objects.",
- "parameters": [
- {
- "name": "page[after]",
- "in": "query",
- "schema": {
- "type": "string",
- "format": "byte"
- },
- "example": "eyJwcmltYXJ5Ijp7ImNvbmZpZ0lkIjogMTIzfX0=",
- "description": "Pointer to paginate to retrieve the data after the object provided. Specify the `base64` encoded JSON string in a **uniquely identifiable** manner (e.g. object IDs), i.e. by using one (primary) or two (primary and secondary) fields that allow for **stable** sorting.\n \n\nFormat: `{\"primary\":{\"someField\": 123},\"secondary\":{\"someOtherOptionalField\": \"Foo\"}}`\n \n\nExample: `{\"primary\":{\"configId\": 123}}` -> `eyJwcmltYXJ5Ijp7ImNvbmZpZ0lkIjogMTIzfX0=`"
- },
- {
- "name": "page[before]",
- "in": "query",
- "schema": {
- "type": "string",
- "format": "byte"
- },
- "example": "eyJwcmltYXJ5Ijp7ImNvbmZpZ0lkIjogMTIzfX0=",
- "description": "Pointer to paginate to retrieve the data before the object provided. Specify the `base64` encoded JSON string in a **uniquely identifiable** manner (e.g. object IDs), i.e. by using one (primary) or two (primary and secondary) fields that allow for **stable** sorting.\n \n\nFormat: `{\"primary\":{\"someField\": 123},\"secondary\":{\"someOtherOptionalField\": \"Foo\"}}`\n \n\nExample: `{\"primary\":{\"configId\": 123}}` -> `eyJwcmltYXJ5Ijp7ImNvbmZpZ0lkIjogMTIzfX0=`"
- },
- {
- "name": "page[size]",
- "in": "query",
- "schema": {
- "type": "integer",
- "format": "int32"
- },
- "example": 100,
- "description": "Amout of data to retrieve inside a single page"
- },
- {
- "name": "filter",
- "in": "query",
- "style": "deepObject",
- "explode": true,
- "schema": {
- "type": "object",
- "additionalProperties": {
- "type": "string"
- }
- },
- "description": "Filters results using a query. Every key is an attribute name optionally suffixed with a comparison operator, e.g. `filter[configId__gt]=200`.",
- "example": {
- "configId__gt": "200"
- }
- },
- {
- "name": "include",
- "in": "query",
- "style": "form",
- "explode": false,
- "schema": {
- "type": "array",
- "items": {
- "type": "string",
- "enum": [
- "configSection"
+ "user"
]
}
},
- "description": "Relationships to include in the response, comma seperated. Possible options: configSection",
+ "description": "Relationships to include in the response, comma seperated. Possible options: user",
"example": [
- "configSection"
+ "user"
]
- },
- {
- "name": "aggregate",
- "in": "query",
- "style": "deepObject",
- "explode": true,
- "schema": {
- "type": "object",
- "additionalProperties": {
- "type": "string"
- }
- },
- "required": false,
- "description": "Aggregated fields to include by type (comma separated values). Possible options: config: valueBoundaries",
- "example": {
- "aggregate[config]": "valueBoundaries"
- }
}
],
- "summary": "List Configs",
- "operationId": "getConfigs"
+ "summary": "Get BackgroundJobs",
+ "operationId": "getBackgroundJobsById"
},
- "patch": {
+ "delete": {
"tags": [
- "Configs"
+ "BackgroundJobs"
],
"responses": {
"400": {
@@ -6795,9 +6719,6 @@
}
}
},
- "204": {
- "description": "successfully updated"
- },
"404": {
"description": "Not Found",
"content": {
@@ -6808,15 +6729,8 @@
}
}
},
- "409": {
- "description": "Resource already exists",
- "content": {
- "application/problem+json": {
- "schema": {
- "$ref": "#/components/schemas/ErrorResponse"
- }
- }
- }
+ "204": {
+ "description": "successfully deleted"
}
},
"security": [
@@ -6824,26 +6738,25 @@
"bearerAuth": []
}
],
- "description": "PATCH request to update attributes of a single object.",
- "requestBody": {
- "required": true,
- "content": {
- "application/vnd.api+json": {
- "schema": {
- "$ref": "#/components/schemas/ConfigPatchMultiple"
- }
+ "description": "Delete BackgroundJobs",
+ "parameters": [
+ {
+ "name": "id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "integer"
}
}
- },
- "parameters": [],
- "summary": "Update Configs",
- "operationId": "patchConfigs"
+ ],
+ "summary": "Delete BackgroundJobs",
+ "operationId": "deleteBackgroundJobsById"
}
},
- "/api/v2/ui/configs/count": {
+ "/api/v2/ui/chunks": {
"get": {
"tags": [
- "Configs"
+ "Chunks"
],
"responses": {
"400": {
@@ -6881,7 +6794,7 @@
"content": {
"application/vnd.api+json": {
"schema": {
- "$ref": "#/components/schemas/ConfigCountResponse"
+ "$ref": "#/components/schemas/ChunkListResponse"
}
}
}
@@ -6894,6 +6807,36 @@
],
"description": "GET many request to retrieve multiple objects.",
"parameters": [
+ {
+ "name": "page[after]",
+ "in": "query",
+ "schema": {
+ "type": "string",
+ "format": "byte"
+ },
+ "example": "eyJwcmltYXJ5Ijp7ImNodW5rSWQiOiAxMjN9fQ==",
+ "description": "Pointer to paginate to retrieve the data after the object provided. Specify the `base64` encoded JSON string in a **uniquely identifiable** manner (e.g. object IDs), i.e. by using one (primary) or two (primary and secondary) fields that allow for **stable** sorting.\n \n\nFormat: `{\"primary\":{\"someField\": 123},\"secondary\":{\"someOtherOptionalField\": \"Foo\"}}`\n \n\nExample: `{\"primary\":{\"chunkId\": 123}}` -> `eyJwcmltYXJ5Ijp7ImNodW5rSWQiOiAxMjN9fQ==`"
+ },
+ {
+ "name": "page[before]",
+ "in": "query",
+ "schema": {
+ "type": "string",
+ "format": "byte"
+ },
+ "example": "eyJwcmltYXJ5Ijp7ImNodW5rSWQiOiAxMjN9fQ==",
+ "description": "Pointer to paginate to retrieve the data before the object provided. Specify the `base64` encoded JSON string in a **uniquely identifiable** manner (e.g. object IDs), i.e. by using one (primary) or two (primary and secondary) fields that allow for **stable** sorting.\n \n\nFormat: `{\"primary\":{\"someField\": 123},\"secondary\":{\"someOtherOptionalField\": \"Foo\"}}`\n \n\nExample: `{\"primary\":{\"chunkId\": 123}}` -> `eyJwcmltYXJ5Ijp7ImNodW5rSWQiOiAxMjN9fQ==`"
+ },
+ {
+ "name": "page[size]",
+ "in": "query",
+ "schema": {
+ "type": "integer",
+ "format": "int32"
+ },
+ "example": 100,
+ "description": "Amout of data to retrieve inside a single page"
+ },
{
"name": "filter",
"in": "query",
@@ -6905,29 +6848,41 @@
"type": "string"
}
},
- "description": "Filters results using a query. Every key is an attribute name optionally suffixed with a comparison operator, e.g. `filter[configId__gt]=200`.",
+ "description": "Filters results using a query. Every key is an attribute name optionally suffixed with a comparison operator, e.g. `filter[chunkId__gt]=200`.",
"example": {
- "configId__gt": "200"
+ "chunkId__gt": "200"
}
},
{
- "name": "include_total",
+ "name": "include",
"in": "query",
+ "style": "form",
+ "explode": false,
"schema": {
- "type": "boolean"
+ "type": "array",
+ "items": {
+ "type": "string",
+ "enum": [
+ "agent",
+ "task"
+ ]
+ }
},
- "example": true,
- "description": "Also report the number of accessible objects without any filter applied, as `meta.total_count`"
+ "description": "Relationships to include in the response, comma seperated. Possible options: agent, task",
+ "example": [
+ "agent",
+ "task"
+ ]
}
],
- "summary": "Count Configs",
- "operationId": "getConfigsCount"
+ "summary": "List Chunks",
+ "operationId": "getChunks"
}
},
- "/api/v2/ui/configs/{id}/{relation}": {
+ "/api/v2/ui/chunks/count": {
"get": {
"tags": [
- "Configs"
+ "Chunks"
],
"responses": {
"400": {
@@ -6960,22 +6915,12 @@
}
}
},
- "404": {
- "description": "Not Found",
- "content": {
- "application/problem+json": {
- "schema": {
- "$ref": "#/components/schemas/ErrorResponse"
- }
- }
- }
- },
"200": {
"description": "successful operation",
"content": {
"application/vnd.api+json": {
"schema": {
- "$ref": "#/components/schemas/ConfigRelationConfigSectionGetResponse"
+ "$ref": "#/components/schemas/ChunkCountResponse"
}
}
}
@@ -6986,35 +6931,42 @@
"bearerAuth": []
}
],
- "description": "GET request to retrieve a single object.",
+ "description": "GET many request to retrieve multiple objects.",
"parameters": [
{
- "name": "id",
- "in": "path",
- "required": true,
+ "name": "filter",
+ "in": "query",
+ "style": "deepObject",
+ "explode": true,
"schema": {
- "type": "integer",
- "format": "int32",
- "example": 10
+ "type": "object",
+ "additionalProperties": {
+ "type": "string"
+ }
+ },
+ "description": "Filters results using a query. Every key is an attribute name optionally suffixed with a comparison operator, e.g. `filter[chunkId__gt]=200`.",
+ "example": {
+ "chunkId__gt": "200"
}
},
{
- "name": "relation",
- "in": "path",
- "required": true,
+ "name": "include_total",
+ "in": "query",
"schema": {
- "type": "string"
- }
+ "type": "boolean"
+ },
+ "example": true,
+ "description": "Also report the number of accessible objects without any filter applied, as `meta.total_count`"
}
],
- "summary": "Get Configs",
- "operationId": "getConfigsByIdByRelation"
+ "summary": "Count Chunks",
+ "operationId": "getChunksCount"
}
},
- "/api/v2/ui/configs/{id}/relationships/{relation}": {
+ "/api/v2/ui/chunks/{id}/{relation}": {
"get": {
"tags": [
- "Configs"
+ "Chunks"
],
"responses": {
"400": {
@@ -7062,7 +7014,7 @@
"content": {
"application/vnd.api+json": {
"schema": {
- "$ref": "#/components/schemas/ConfigResponse"
+ "$ref": "#/components/schemas/ChunkRelationTaskGetResponse"
}
}
}
@@ -7073,7 +7025,7 @@
"bearerAuth": []
}
],
- "description": "GET request for for a to-one relationship link. Returns the resource record of the object that is part of the specified relation.",
+ "description": "GET request to retrieve a single object.",
"parameters": [
{
"name": "id",
@@ -7094,12 +7046,14 @@
}
}
],
- "summary": "Get Configs",
- "operationId": "getConfigsByIdRelationshipsByRelation"
- },
- "patch": {
+ "summary": "Get Chunks",
+ "operationId": "getChunksByIdByRelation"
+ }
+ },
+ "/api/v2/ui/chunks/{id}/relationships/{relation}": {
+ "get": {
"tags": [
- "Configs"
+ "Chunks"
],
"responses": {
"400": {
@@ -7132,9 +7086,6 @@
}
}
},
- "204": {
- "description": "Successfull operation"
- },
"404": {
"description": "Not Found",
"content": {
@@ -7145,12 +7096,12 @@
}
}
},
- "409": {
- "description": "Resource already exists",
+ "200": {
+ "description": "successful operation",
"content": {
- "application/problem+json": {
+ "application/vnd.api+json": {
"schema": {
- "$ref": "#/components/schemas/ErrorResponse"
+ "$ref": "#/components/schemas/ChunkResponse"
}
}
}
@@ -7161,24 +7112,16 @@
"bearerAuth": []
}
],
- "description": "PATCH request to update a to one relationship.",
- "requestBody": {
- "required": true,
- "content": {
- "application/vnd.api+json": {
- "schema": {
- "$ref": "#/components/schemas/ConfigRelationConfigSection"
- }
- }
- }
- },
+ "description": "GET request for for a to-one relationship link. Returns the resource record of the object that is part of the specified relation.",
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"schema": {
- "type": "integer"
+ "type": "integer",
+ "format": "int32",
+ "example": 10
}
},
{
@@ -7190,14 +7133,12 @@
}
}
],
- "summary": "Update Configs",
- "operationId": "patchConfigsByIdRelationshipsByRelation"
- }
- },
- "/api/v2/ui/configs/{id}": {
- "get": {
+ "summary": "Get Chunks",
+ "operationId": "getChunksByIdRelationshipsByRelation"
+ },
+ "patch": {
"tags": [
- "Configs"
+ "Chunks"
],
"responses": {
"400": {
@@ -7230,6 +7171,9 @@
}
}
},
+ "204": {
+ "description": "Successfull operation"
+ },
"404": {
"description": "Not Found",
"content": {
@@ -7240,12 +7184,12 @@
}
}
},
- "200": {
- "description": "successful operation",
+ "409": {
+ "description": "Resource already exists",
"content": {
- "application/vnd.api+json": {
+ "application/problem+json": {
"schema": {
- "$ref": "#/components/schemas/ConfigResponse"
+ "$ref": "#/components/schemas/ErrorResponse"
}
}
}
@@ -7256,44 +7200,813 @@
"bearerAuth": []
}
],
- "description": "GET request to retrieve a single object.",
+ "description": "PATCH request to update a to one relationship.",
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/vnd.api+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ChunkRelationTask"
+ }
+ }
+ }
+ },
"parameters": [
{
"name": "id",
"in": "path",
"required": true,
"schema": {
- "type": "integer",
- "format": "int32",
- "example": 10
+ "type": "integer"
}
},
{
- "name": "include",
- "in": "query",
- "style": "form",
- "explode": false,
+ "name": "relation",
+ "in": "path",
+ "required": true,
"schema": {
- "type": "array",
- "items": {
- "type": "string",
- "enum": [
- "configSection"
- ]
- }
- },
- "description": "Relationships to include in the response, comma seperated. Possible options: configSection",
- "example": [
- "configSection"
- ]
+ "type": "string"
+ }
}
],
- "summary": "Get Configs",
- "operationId": "getConfigsById"
- },
- "patch": {
+ "summary": "Update Chunks",
+ "operationId": "patchChunksByIdRelationshipsByRelation"
+ }
+ },
+ "/api/v2/ui/chunks/{id}": {
+ "get": {
"tags": [
- "Configs"
+ "Chunks"
+ ],
+ "responses": {
+ "400": {
+ "description": "Invalid request",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "401": {
+ "description": "Authentication failed",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "403": {
+ "description": "Permission denied",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "404": {
+ "description": "Not Found",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "200": {
+ "description": "successful operation",
+ "content": {
+ "application/vnd.api+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ChunkResponse"
+ }
+ }
+ }
+ }
+ },
+ "security": [
+ {
+ "bearerAuth": []
+ }
+ ],
+ "description": "GET request to retrieve a single object.",
+ "parameters": [
+ {
+ "name": "id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "integer",
+ "format": "int32",
+ "example": 10
+ }
+ },
+ {
+ "name": "include",
+ "in": "query",
+ "style": "form",
+ "explode": false,
+ "schema": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "enum": [
+ "agent",
+ "task"
+ ]
+ }
+ },
+ "description": "Relationships to include in the response, comma seperated. Possible options: agent, task",
+ "example": [
+ "agent",
+ "task"
+ ]
+ }
+ ],
+ "summary": "Get Chunks",
+ "operationId": "getChunksById"
+ }
+ },
+ "/api/v2/ui/configs": {
+ "get": {
+ "tags": [
+ "Configs"
+ ],
+ "responses": {
+ "400": {
+ "description": "Invalid request",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "401": {
+ "description": "Authentication failed",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "403": {
+ "description": "Permission denied",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "200": {
+ "description": "successful operation",
+ "content": {
+ "application/vnd.api+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ConfigListResponse"
+ }
+ }
+ }
+ }
+ },
+ "security": [
+ {
+ "bearerAuth": []
+ }
+ ],
+ "description": "GET many request to retrieve multiple objects.",
+ "parameters": [
+ {
+ "name": "page[after]",
+ "in": "query",
+ "schema": {
+ "type": "string",
+ "format": "byte"
+ },
+ "example": "eyJwcmltYXJ5Ijp7ImNvbmZpZ0lkIjogMTIzfX0=",
+ "description": "Pointer to paginate to retrieve the data after the object provided. Specify the `base64` encoded JSON string in a **uniquely identifiable** manner (e.g. object IDs), i.e. by using one (primary) or two (primary and secondary) fields that allow for **stable** sorting.\n \n\nFormat: `{\"primary\":{\"someField\": 123},\"secondary\":{\"someOtherOptionalField\": \"Foo\"}}`\n \n\nExample: `{\"primary\":{\"configId\": 123}}` -> `eyJwcmltYXJ5Ijp7ImNvbmZpZ0lkIjogMTIzfX0=`"
+ },
+ {
+ "name": "page[before]",
+ "in": "query",
+ "schema": {
+ "type": "string",
+ "format": "byte"
+ },
+ "example": "eyJwcmltYXJ5Ijp7ImNvbmZpZ0lkIjogMTIzfX0=",
+ "description": "Pointer to paginate to retrieve the data before the object provided. Specify the `base64` encoded JSON string in a **uniquely identifiable** manner (e.g. object IDs), i.e. by using one (primary) or two (primary and secondary) fields that allow for **stable** sorting.\n \n\nFormat: `{\"primary\":{\"someField\": 123},\"secondary\":{\"someOtherOptionalField\": \"Foo\"}}`\n \n\nExample: `{\"primary\":{\"configId\": 123}}` -> `eyJwcmltYXJ5Ijp7ImNvbmZpZ0lkIjogMTIzfX0=`"
+ },
+ {
+ "name": "page[size]",
+ "in": "query",
+ "schema": {
+ "type": "integer",
+ "format": "int32"
+ },
+ "example": 100,
+ "description": "Amout of data to retrieve inside a single page"
+ },
+ {
+ "name": "filter",
+ "in": "query",
+ "style": "deepObject",
+ "explode": true,
+ "schema": {
+ "type": "object",
+ "additionalProperties": {
+ "type": "string"
+ }
+ },
+ "description": "Filters results using a query. Every key is an attribute name optionally suffixed with a comparison operator, e.g. `filter[configId__gt]=200`.",
+ "example": {
+ "configId__gt": "200"
+ }
+ },
+ {
+ "name": "include",
+ "in": "query",
+ "style": "form",
+ "explode": false,
+ "schema": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "enum": [
+ "configSection"
+ ]
+ }
+ },
+ "description": "Relationships to include in the response, comma seperated. Possible options: configSection",
+ "example": [
+ "configSection"
+ ]
+ },
+ {
+ "name": "aggregate",
+ "in": "query",
+ "style": "deepObject",
+ "explode": true,
+ "schema": {
+ "type": "object",
+ "additionalProperties": {
+ "type": "string"
+ }
+ },
+ "required": false,
+ "description": "Aggregated fields to include by type (comma separated values). Possible options: config: valueBoundaries",
+ "example": {
+ "aggregate[config]": "valueBoundaries"
+ }
+ }
+ ],
+ "summary": "List Configs",
+ "operationId": "getConfigs"
+ },
+ "patch": {
+ "tags": [
+ "Configs"
+ ],
+ "responses": {
+ "400": {
+ "description": "Invalid request",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "401": {
+ "description": "Authentication failed",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "403": {
+ "description": "Permission denied",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "204": {
+ "description": "successfully updated"
+ },
+ "404": {
+ "description": "Not Found",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "409": {
+ "description": "Resource already exists",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ }
+ },
+ "security": [
+ {
+ "bearerAuth": []
+ }
+ ],
+ "description": "PATCH request to update attributes of a single object.",
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/vnd.api+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ConfigPatchMultiple"
+ }
+ }
+ }
+ },
+ "parameters": [],
+ "summary": "Update Configs",
+ "operationId": "patchConfigs"
+ }
+ },
+ "/api/v2/ui/configs/count": {
+ "get": {
+ "tags": [
+ "Configs"
+ ],
+ "responses": {
+ "400": {
+ "description": "Invalid request",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "401": {
+ "description": "Authentication failed",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "403": {
+ "description": "Permission denied",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "200": {
+ "description": "successful operation",
+ "content": {
+ "application/vnd.api+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ConfigCountResponse"
+ }
+ }
+ }
+ }
+ },
+ "security": [
+ {
+ "bearerAuth": []
+ }
+ ],
+ "description": "GET many request to retrieve multiple objects.",
+ "parameters": [
+ {
+ "name": "filter",
+ "in": "query",
+ "style": "deepObject",
+ "explode": true,
+ "schema": {
+ "type": "object",
+ "additionalProperties": {
+ "type": "string"
+ }
+ },
+ "description": "Filters results using a query. Every key is an attribute name optionally suffixed with a comparison operator, e.g. `filter[configId__gt]=200`.",
+ "example": {
+ "configId__gt": "200"
+ }
+ },
+ {
+ "name": "include_total",
+ "in": "query",
+ "schema": {
+ "type": "boolean"
+ },
+ "example": true,
+ "description": "Also report the number of accessible objects without any filter applied, as `meta.total_count`"
+ }
+ ],
+ "summary": "Count Configs",
+ "operationId": "getConfigsCount"
+ }
+ },
+ "/api/v2/ui/configs/{id}/{relation}": {
+ "get": {
+ "tags": [
+ "Configs"
+ ],
+ "responses": {
+ "400": {
+ "description": "Invalid request",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "401": {
+ "description": "Authentication failed",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "403": {
+ "description": "Permission denied",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "404": {
+ "description": "Not Found",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "200": {
+ "description": "successful operation",
+ "content": {
+ "application/vnd.api+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ConfigRelationConfigSectionGetResponse"
+ }
+ }
+ }
+ }
+ },
+ "security": [
+ {
+ "bearerAuth": []
+ }
+ ],
+ "description": "GET request to retrieve a single object.",
+ "parameters": [
+ {
+ "name": "id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "integer",
+ "format": "int32",
+ "example": 10
+ }
+ },
+ {
+ "name": "relation",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string"
+ }
+ }
+ ],
+ "summary": "Get Configs",
+ "operationId": "getConfigsByIdByRelation"
+ }
+ },
+ "/api/v2/ui/configs/{id}/relationships/{relation}": {
+ "get": {
+ "tags": [
+ "Configs"
+ ],
+ "responses": {
+ "400": {
+ "description": "Invalid request",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "401": {
+ "description": "Authentication failed",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "403": {
+ "description": "Permission denied",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "404": {
+ "description": "Not Found",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "200": {
+ "description": "successful operation",
+ "content": {
+ "application/vnd.api+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ConfigResponse"
+ }
+ }
+ }
+ }
+ },
+ "security": [
+ {
+ "bearerAuth": []
+ }
+ ],
+ "description": "GET request for for a to-one relationship link. Returns the resource record of the object that is part of the specified relation.",
+ "parameters": [
+ {
+ "name": "id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "integer",
+ "format": "int32",
+ "example": 10
+ }
+ },
+ {
+ "name": "relation",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string"
+ }
+ }
+ ],
+ "summary": "Get Configs",
+ "operationId": "getConfigsByIdRelationshipsByRelation"
+ },
+ "patch": {
+ "tags": [
+ "Configs"
+ ],
+ "responses": {
+ "400": {
+ "description": "Invalid request",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "401": {
+ "description": "Authentication failed",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "403": {
+ "description": "Permission denied",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "204": {
+ "description": "Successfull operation"
+ },
+ "404": {
+ "description": "Not Found",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "409": {
+ "description": "Resource already exists",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ }
+ },
+ "security": [
+ {
+ "bearerAuth": []
+ }
+ ],
+ "description": "PATCH request to update a to one relationship.",
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/vnd.api+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ConfigRelationConfigSection"
+ }
+ }
+ }
+ },
+ "parameters": [
+ {
+ "name": "id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "integer"
+ }
+ },
+ {
+ "name": "relation",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "string"
+ }
+ }
+ ],
+ "summary": "Update Configs",
+ "operationId": "patchConfigsByIdRelationshipsByRelation"
+ }
+ },
+ "/api/v2/ui/configs/{id}": {
+ "get": {
+ "tags": [
+ "Configs"
+ ],
+ "responses": {
+ "400": {
+ "description": "Invalid request",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "401": {
+ "description": "Authentication failed",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "403": {
+ "description": "Permission denied",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "404": {
+ "description": "Not Found",
+ "content": {
+ "application/problem+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ErrorResponse"
+ }
+ }
+ }
+ },
+ "200": {
+ "description": "successful operation",
+ "content": {
+ "application/vnd.api+json": {
+ "schema": {
+ "$ref": "#/components/schemas/ConfigResponse"
+ }
+ }
+ }
+ }
+ },
+ "security": [
+ {
+ "bearerAuth": []
+ }
+ ],
+ "description": "GET request to retrieve a single object.",
+ "parameters": [
+ {
+ "name": "id",
+ "in": "path",
+ "required": true,
+ "schema": {
+ "type": "integer",
+ "format": "int32",
+ "example": 10
+ }
+ },
+ {
+ "name": "include",
+ "in": "query",
+ "style": "form",
+ "explode": false,
+ "schema": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "enum": [
+ "configSection"
+ ]
+ }
+ },
+ "description": "Relationships to include in the response, comma seperated. Possible options: configSection",
+ "example": [
+ "configSection"
+ ]
+ }
+ ],
+ "summary": "Get Configs",
+ "operationId": "getConfigsById"
+ },
+ "patch": {
+ "tags": [
+ "Configs"
],
"responses": {
"400": {
@@ -28312,7 +29025,7 @@
},
"/api/v2/helper/recountFileLines": {
"post": {
- "description": "Endpoint to recount files for when there is size mismatch",
+ "description": "Endpoint to enqueue the recounting of files for when there is size mismatch. The line count is
calculated asynchronously by the background job runner.",
"responses": {
"400": {
"description": "Invalid request",
@@ -37796,7 +38509,636 @@
}
}
},
- "ApiTokenPostPatchResponse": {
+ "ApiTokenPostPatchResponse": {
+ "type": "object",
+ "required": [
+ "jsonapi",
+ "links",
+ "data"
+ ],
+ "properties": {
+ "jsonapi": {
+ "type": "object",
+ "required": [
+ "version"
+ ],
+ "properties": {
+ "version": {
+ "type": "string",
+ "default": "1.1"
+ },
+ "ext": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "default": [
+ "https://jsonapi.org/profiles/ethanresnick/cursor-pagination"
+ ]
+ }
+ }
+ },
+ "links": {
+ "type": "object",
+ "required": [
+ "self"
+ ],
+ "properties": {
+ "self": {
+ "type": "string",
+ "default": "/api/v2/ui/apiTokens/1"
+ }
+ }
+ },
+ "data": {
+ "type": "object",
+ "required": [
+ "id",
+ "type",
+ "attributes",
+ "links",
+ "relationships"
+ ],
+ "properties": {
+ "id": {
+ "type": "integer",
+ "example": 1
+ },
+ "type": {
+ "type": "string",
+ "const": "apiToken"
+ },
+ "attributes": {
+ "type": "object",
+ "required": [
+ "startValid",
+ "endValid",
+ "userId",
+ "tokenName",
+ "isRevoked"
+ ],
+ "properties": {
+ "startValid": {
+ "type": "integer",
+ "format": "int64"
+ },
+ "endValid": {
+ "type": "integer",
+ "format": "int64"
+ },
+ "userId": {
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "tokenName": {
+ "type": "string"
+ },
+ "isRevoked": {
+ "type": "boolean"
+ },
+ "token": {
+ "type": "string"
+ }
+ }
+ },
+ "links": {
+ "type": "object",
+ "required": [
+ "self"
+ ],
+ "properties": {
+ "self": {
+ "type": "string",
+ "default": "/api/v2/ui/apiTokens/1"
+ }
+ }
+ },
+ "relationships": {
+ "type": "object",
+ "required": [
+ "user"
+ ],
+ "properties": {
+ "user": {
+ "type": "object",
+ "required": [
+ "links"
+ ],
+ "properties": {
+ "links": {
+ "type": "object",
+ "required": [
+ "self",
+ "related"
+ ],
+ "properties": {
+ "self": {
+ "type": "string",
+ "default": "/api/v2/ui/apiTokens/relationships/user"
+ },
+ "related": {
+ "type": "string",
+ "default": "/api/v2/ui/apiTokens/user"
+ }
+ }
+ },
+ "data": {
+ "oneOf": [
+ {
+ "type": "object",
+ "required": [
+ "type",
+ "id"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "const": "user"
+ },
+ "id": {
+ "type": "integer",
+ "example": 1
+ }
+ }
+ },
+ {
+ "type": "null"
+ }
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "included": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [
+ "id",
+ "type",
+ "attributes"
+ ],
+ "properties": {
+ "id": {
+ "type": "integer",
+ "example": 1
+ },
+ "type": {
+ "type": "string",
+ "const": "user"
+ },
+ "attributes": {
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ },
+ "isValid": {
+ "type": "boolean"
+ },
+ "isComputedPassword": {
+ "type": "boolean"
+ },
+ "lastLoginDate": {
+ "type": "integer",
+ "format": "int64"
+ },
+ "registeredSince": {
+ "type": "integer",
+ "format": "int64"
+ },
+ "sessionLifetime": {
+ "type": "integer"
+ },
+ "globalPermissionGroupId": {
+ "type": "integer"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "ApiTokenListResponse": {
+ "type": "object",
+ "required": [
+ "jsonapi",
+ "links",
+ "meta",
+ "data"
+ ],
+ "properties": {
+ "jsonapi": {
+ "type": "object",
+ "required": [
+ "version"
+ ],
+ "properties": {
+ "version": {
+ "type": "string",
+ "default": "1.1"
+ },
+ "ext": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "default": [
+ "https://jsonapi.org/profiles/ethanresnick/cursor-pagination"
+ ]
+ }
+ }
+ },
+ "links": {
+ "type": "object",
+ "required": [
+ "self",
+ "first",
+ "last",
+ "next",
+ "prev"
+ ],
+ "properties": {
+ "self": {
+ "type": "string",
+ "default": "/api/v2/ui/apiTokens?page[size]=25"
+ },
+ "first": {
+ "type": "string",
+ "default": "/api/v2/ui/apiTokens?page[size]=25"
+ },
+ "last": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "default": "/api/v2/ui/apiTokens?page[size]=25&page[before]=eyJwcmltYXJ5Ijp7InNvbWVVbnFpdWVGaWVsZCI6MTIzfSwic2Vjb25kYXJ5Ijp7InNvbWVPdGhlck9wdGlvbmFsRmllbGQiOiJGb28ifX0="
+ },
+ "next": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "default": "/api/v2/ui/apiTokens?page[size]=25&page[after]=eyJwcmltYXJ5Ijp7InNvbWVVbnFpdWVGaWVsZCI6MTIzfSwic2Vjb25kYXJ5Ijp7InNvbWVPdGhlck9wdGlvbmFsRmllbGQiOiJGb28ifX0="
+ },
+ "prev": {
+ "type": [
+ "string",
+ "null"
+ ],
+ "default": "/api/v2/ui/apiTokens?page[size]=25&page[before]=eyJwcmltYXJ5Ijp7InNvbWVVbnFpdWVGaWVsZCI6MTIzfSwic2Vjb25kYXJ5Ijp7InNvbWVPdGhlck9wdGlvbmFsRmllbGQiOiJGb28ifX0="
+ }
+ }
+ },
+ "meta": {
+ "type": "object",
+ "required": [
+ "page"
+ ],
+ "properties": {
+ "page": {
+ "type": "object",
+ "required": [
+ "total_elements"
+ ],
+ "properties": {
+ "total_elements": {
+ "type": "integer"
+ }
+ }
+ }
+ }
+ },
+ "data": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [
+ "id",
+ "type",
+ "attributes",
+ "links",
+ "relationships"
+ ],
+ "properties": {
+ "id": {
+ "type": "integer",
+ "example": 1
+ },
+ "type": {
+ "type": "string",
+ "const": "apiToken"
+ },
+ "attributes": {
+ "type": "object",
+ "required": [
+ "startValid",
+ "endValid",
+ "userId",
+ "tokenName",
+ "isRevoked"
+ ],
+ "properties": {
+ "startValid": {
+ "type": "integer",
+ "format": "int64"
+ },
+ "endValid": {
+ "type": "integer",
+ "format": "int64"
+ },
+ "userId": {
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "tokenName": {
+ "type": "string"
+ },
+ "isRevoked": {
+ "type": "boolean"
+ },
+ "token": {
+ "type": "string"
+ }
+ }
+ },
+ "links": {
+ "type": "object",
+ "required": [
+ "self"
+ ],
+ "properties": {
+ "self": {
+ "type": "string",
+ "default": "/api/v2/ui/apiTokens/1"
+ }
+ }
+ },
+ "relationships": {
+ "type": "object",
+ "required": [
+ "user"
+ ],
+ "properties": {
+ "user": {
+ "type": "object",
+ "required": [
+ "links"
+ ],
+ "properties": {
+ "links": {
+ "type": "object",
+ "required": [
+ "self",
+ "related"
+ ],
+ "properties": {
+ "self": {
+ "type": "string",
+ "default": "/api/v2/ui/apiTokens/relationships/user"
+ },
+ "related": {
+ "type": "string",
+ "default": "/api/v2/ui/apiTokens/user"
+ }
+ }
+ },
+ "data": {
+ "oneOf": [
+ {
+ "type": "object",
+ "required": [
+ "type",
+ "id"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "const": "user"
+ },
+ "id": {
+ "type": "integer",
+ "example": 1
+ }
+ }
+ },
+ {
+ "type": "null"
+ }
+ ]
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "included": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [
+ "id",
+ "type",
+ "attributes"
+ ],
+ "properties": {
+ "id": {
+ "type": "integer",
+ "example": 1
+ },
+ "type": {
+ "type": "string",
+ "const": "user"
+ },
+ "attributes": {
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string"
+ },
+ "email": {
+ "type": "string"
+ },
+ "isValid": {
+ "type": "boolean"
+ },
+ "isComputedPassword": {
+ "type": "boolean"
+ },
+ "lastLoginDate": {
+ "type": "integer",
+ "format": "int64"
+ },
+ "registeredSince": {
+ "type": "integer",
+ "format": "int64"
+ },
+ "sessionLifetime": {
+ "type": "integer"
+ },
+ "globalPermissionGroupId": {
+ "type": "integer"
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ },
+ "ApiTokenCountResponse": {
+ "type": "object",
+ "required": [
+ "jsonapi",
+ "meta",
+ "data"
+ ],
+ "properties": {
+ "jsonapi": {
+ "type": "object",
+ "required": [
+ "version"
+ ],
+ "properties": {
+ "version": {
+ "type": "string",
+ "default": "1.1"
+ },
+ "ext": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "default": [
+ "https://jsonapi.org/profiles/ethanresnick/cursor-pagination"
+ ]
+ }
+ }
+ },
+ "meta": {
+ "type": "object",
+ "required": [
+ "count"
+ ],
+ "properties": {
+ "count": {
+ "type": "integer",
+ "description": "Number of objects accessible to the current user matching the given filters"
+ },
+ "total_count": {
+ "type": "integer",
+ "description": "Number of objects accessible to the current user without any filter applied, only present when `include_total=true` was requested"
+ }
+ }
+ },
+ "data": {
+ "type": "array",
+ "items": {
+ "type": "object"
+ },
+ "maxItems": 0,
+ "description": "Always empty: the count is reported under meta."
+ }
+ }
+ },
+ "ApiTokenRelationUser": {
+ "type": "object",
+ "required": [
+ "data"
+ ],
+ "properties": {
+ "data": {
+ "type": "object",
+ "required": [
+ "type",
+ "id"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "const": "user"
+ },
+ "id": {
+ "type": "integer",
+ "example": 1
+ }
+ }
+ }
+ }
+ },
+ "ApiTokenRelationUserGetResponse": {
+ "type": "object",
+ "required": [
+ "data"
+ ],
+ "properties": {
+ "data": {
+ "type": "object",
+ "required": [
+ "type",
+ "id"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "const": "user"
+ },
+ "id": {
+ "type": "integer",
+ "example": 1
+ }
+ }
+ }
+ }
+ },
+ "BackgroundJobDeleteMultiple": {
+ "type": "object",
+ "required": [
+ "data"
+ ],
+ "properties": {
+ "data": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "required": [
+ "id",
+ "type"
+ ],
+ "properties": {
+ "id": {
+ "type": "integer",
+ "example": 1
+ },
+ "type": {
+ "type": "string",
+ "const": "backgroundJob"
+ }
+ }
+ }
+ }
+ }
+ },
+ "BackgroundJobResponse": {
"type": "object",
"required": [
"jsonapi",
@@ -37833,7 +39175,7 @@
"properties": {
"self": {
"type": "string",
- "default": "/api/v2/ui/apiTokens/1"
+ "default": "/api/v2/ui/backgroundJobs/1"
}
}
},
@@ -37853,25 +39195,51 @@
},
"type": {
"type": "string",
- "const": "apiToken"
+ "const": "backgroundJob"
},
"attributes": {
"type": "object",
"required": [
- "startValid",
- "endValid",
+ "jobType",
+ "payload",
+ "status",
"userId",
- "tokenName",
- "isRevoked"
+ "createdAt",
+ "startedAt",
+ "finishedAt",
+ "exitCode",
+ "resultMessage"
],
"properties": {
- "startValid": {
- "type": "integer",
- "format": "int64"
+ "jobType": {
+ "type": "string"
},
- "endValid": {
- "type": "integer",
- "format": "int64"
+ "payload": {
+ "type": "object"
+ },
+ "status": {
+ "oneOf": [
+ {
+ "const": -1,
+ "title": "Failed",
+ "type": "integer"
+ },
+ {
+ "const": 0,
+ "title": "Pending",
+ "type": "integer"
+ },
+ {
+ "const": 1,
+ "title": "Running",
+ "type": "integer"
+ },
+ {
+ "const": 2,
+ "title": "Done",
+ "type": "integer"
+ }
+ ]
},
"userId": {
"type": [
@@ -37879,14 +39247,35 @@
"null"
]
},
- "tokenName": {
- "type": "string"
+ "createdAt": {
+ "type": "integer",
+ "format": "int64"
},
- "isRevoked": {
- "type": "boolean"
+ "startedAt": {
+ "type": [
+ "integer",
+ "null"
+ ],
+ "format": "int64"
},
- "token": {
- "type": "string"
+ "finishedAt": {
+ "type": [
+ "integer",
+ "null"
+ ],
+ "format": "int64"
+ },
+ "exitCode": {
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "resultMessage": {
+ "type": [
+ "string",
+ "null"
+ ]
}
}
},
@@ -37898,7 +39287,7 @@
"properties": {
"self": {
"type": "string",
- "default": "/api/v2/ui/apiTokens/1"
+ "default": "/api/v2/ui/backgroundJobs/1"
}
}
},
@@ -37923,11 +39312,11 @@
"properties": {
"self": {
"type": "string",
- "default": "/api/v2/ui/apiTokens/relationships/user"
+ "default": "/api/v2/ui/backgroundJobs/relationships/user"
},
"related": {
"type": "string",
- "default": "/api/v2/ui/apiTokens/user"
+ "default": "/api/v2/ui/backgroundJobs/user"
}
}
},
@@ -38018,7 +39407,7 @@
}
}
},
- "ApiTokenListResponse": {
+ "BackgroundJobListResponse": {
"type": "object",
"required": [
"jsonapi",
@@ -38060,32 +39449,32 @@
"properties": {
"self": {
"type": "string",
- "default": "/api/v2/ui/apiTokens?page[size]=25"
+ "default": "/api/v2/ui/backgroundJobs?page[size]=25"
},
"first": {
"type": "string",
- "default": "/api/v2/ui/apiTokens?page[size]=25"
+ "default": "/api/v2/ui/backgroundJobs?page[size]=25"
},
"last": {
"type": [
"string",
"null"
],
- "default": "/api/v2/ui/apiTokens?page[size]=25&page[before]=eyJwcmltYXJ5Ijp7InNvbWVVbnFpdWVGaWVsZCI6MTIzfSwic2Vjb25kYXJ5Ijp7InNvbWVPdGhlck9wdGlvbmFsRmllbGQiOiJGb28ifX0="
+ "default": "/api/v2/ui/backgroundJobs?page[size]=25&page[before]=eyJwcmltYXJ5Ijp7InNvbWVVbnFpdWVGaWVsZCI6MTIzfSwic2Vjb25kYXJ5Ijp7InNvbWVPdGhlck9wdGlvbmFsRmllbGQiOiJGb28ifX0="
},
"next": {
"type": [
"string",
"null"
],
- "default": "/api/v2/ui/apiTokens?page[size]=25&page[after]=eyJwcmltYXJ5Ijp7InNvbWVVbnFpdWVGaWVsZCI6MTIzfSwic2Vjb25kYXJ5Ijp7InNvbWVPdGhlck9wdGlvbmFsRmllbGQiOiJGb28ifX0="
+ "default": "/api/v2/ui/backgroundJobs?page[size]=25&page[after]=eyJwcmltYXJ5Ijp7InNvbWVVbnFpdWVGaWVsZCI6MTIzfSwic2Vjb25kYXJ5Ijp7InNvbWVPdGhlck9wdGlvbmFsRmllbGQiOiJGb28ifX0="
},
"prev": {
"type": [
"string",
"null"
],
- "default": "/api/v2/ui/apiTokens?page[size]=25&page[before]=eyJwcmltYXJ5Ijp7InNvbWVVbnFpdWVGaWVsZCI6MTIzfSwic2Vjb25kYXJ5Ijp7InNvbWVPdGhlck9wdGlvbmFsRmllbGQiOiJGb28ifX0="
+ "default": "/api/v2/ui/backgroundJobs?page[size]=25&page[before]=eyJwcmltYXJ5Ijp7InNvbWVVbnFpdWVGaWVsZCI6MTIzfSwic2Vjb25kYXJ5Ijp7InNvbWVPdGhlck9wdGlvbmFsRmllbGQiOiJGb28ifX0="
}
}
},
@@ -38126,25 +39515,51 @@
},
"type": {
"type": "string",
- "const": "apiToken"
+ "const": "backgroundJob"
},
"attributes": {
"type": "object",
"required": [
- "startValid",
- "endValid",
+ "jobType",
+ "payload",
+ "status",
"userId",
- "tokenName",
- "isRevoked"
+ "createdAt",
+ "startedAt",
+ "finishedAt",
+ "exitCode",
+ "resultMessage"
],
"properties": {
- "startValid": {
- "type": "integer",
- "format": "int64"
+ "jobType": {
+ "type": "string"
},
- "endValid": {
- "type": "integer",
- "format": "int64"
+ "payload": {
+ "type": "object"
+ },
+ "status": {
+ "oneOf": [
+ {
+ "const": -1,
+ "title": "Failed",
+ "type": "integer"
+ },
+ {
+ "const": 0,
+ "title": "Pending",
+ "type": "integer"
+ },
+ {
+ "const": 1,
+ "title": "Running",
+ "type": "integer"
+ },
+ {
+ "const": 2,
+ "title": "Done",
+ "type": "integer"
+ }
+ ]
},
"userId": {
"type": [
@@ -38152,14 +39567,35 @@
"null"
]
},
- "tokenName": {
- "type": "string"
+ "createdAt": {
+ "type": "integer",
+ "format": "int64"
},
- "isRevoked": {
- "type": "boolean"
+ "startedAt": {
+ "type": [
+ "integer",
+ "null"
+ ],
+ "format": "int64"
},
- "token": {
- "type": "string"
+ "finishedAt": {
+ "type": [
+ "integer",
+ "null"
+ ],
+ "format": "int64"
+ },
+ "exitCode": {
+ "type": [
+ "integer",
+ "null"
+ ]
+ },
+ "resultMessage": {
+ "type": [
+ "string",
+ "null"
+ ]
}
}
},
@@ -38171,7 +39607,7 @@
"properties": {
"self": {
"type": "string",
- "default": "/api/v2/ui/apiTokens/1"
+ "default": "/api/v2/ui/backgroundJobs/1"
}
}
},
@@ -38196,11 +39632,11 @@
"properties": {
"self": {
"type": "string",
- "default": "/api/v2/ui/apiTokens/relationships/user"
+ "default": "/api/v2/ui/backgroundJobs/relationships/user"
},
"related": {
"type": "string",
- "default": "/api/v2/ui/apiTokens/user"
+ "default": "/api/v2/ui/backgroundJobs/user"
}
}
},
@@ -38292,7 +39728,7 @@
}
}
},
- "ApiTokenCountResponse": {
+ "BackgroundJobCountResponse": {
"type": "object",
"required": [
"jsonapi",
@@ -38347,7 +39783,7 @@
}
}
},
- "ApiTokenRelationUser": {
+ "BackgroundJobRelationUser": {
"type": "object",
"required": [
"data"
@@ -38372,7 +39808,7 @@
}
}
},
- "ApiTokenRelationUserGetResponse": {
+ "BackgroundJobRelationUserGetResponse": {
"type": "object",
"required": [
"data"
@@ -72319,6 +73755,10 @@
"name": "ApiTokens",
"description": "Reading and writing the resources served under `/api/v2/ui/apiTokens`."
},
+ {
+ "name": "BackgroundJobs",
+ "description": "Reading and writing the resources served under `/api/v2/ui/backgroundJobs`."
+ },
{
"name": "Chunks",
"description": "Reading and writing the resources served under `/api/v2/ui/chunks`."
diff --git a/src/dba/AbstractModelFactory.php b/src/dba/AbstractModelFactory.php
index 19ff07188..c7c67afe3 100755
--- a/src/dba/AbstractModelFactory.php
+++ b/src/dba/AbstractModelFactory.php
@@ -107,14 +107,28 @@ private static function isBinaryColumn(AbstractModel $model, string $key): bool
/**
* @param AbstractModel $model
* @param string $key unmapped column name
- * @return string placeholder SQL fragment ("?" or db-specific hex-to-binary function)
+ * @return bool
*/
- private static function binaryPlaceholder(AbstractModel $model, string $key): string {
- if (!self::isBinaryColumn($model, $key)) {
- return "?";
+ private static function isJsonColumn(AbstractModel $model, string $key): bool {
+ $features = $model->getFeatures();
+ return isset($features[$key]['type']) && $features[$key]['type'] === 'json';
+ }
+
+ /**
+ * @param AbstractModel $model
+ * @param string $key unmapped column name
+ * @return string placeholder SQL fragment ("?" or db-specific value conversion function)
+ */
+ private static function columnPlaceholder(AbstractModel $model, string $key): string {
+ if (self::isBinaryColumn($model, $key)) {
+ $dbType = StartupConfig::getInstance()->getDatabaseType();
+ return $dbType === 'mysql' ? "UNHEX(?)" : "decode(?, 'hex')";
}
- $dbType = StartupConfig::getInstance()->getDatabaseType();
- return $dbType === 'mysql' ? "UNHEX(?)" : "decode(?, 'hex')";
+ elseif (self::isJsonColumn($model, $key)) {
+ $dbType = StartupConfig::getInstance()->getDatabaseType();
+ return $dbType === 'mysql' ? "?" : "?::json";
+ }
+ return "?";
}
/**
@@ -204,7 +218,7 @@ public function save(AbstractModel $model): ?AbstractModel {
$query .= " (" . implode(",", $keys) . ") ";
$placeholders = [];
foreach ($origKeys as $k) {
- $placeholders[] = self::binaryPlaceholder($model, $k);
+ $placeholders[] = self::columnPlaceholder($model, $k);
}
$query = $query . " VALUES (" . implode(",", $placeholders) . ")";
@@ -278,7 +292,7 @@ public function update(AbstractModel $model): PDOStatement {
$mappedKeys = self::getMappedModelKeys($model);
for ($i = 0; $i < count($mappedKeys); $i++) {
- $query .= $mappedKeys[$i] . "=" . self::binaryPlaceholder($model, $origKeys[$i]);
+ $query .= $mappedKeys[$i] . "=" . self::columnPlaceholder($model, $origKeys[$i]);
if ($i < count($mappedKeys) - 1) {
$query .= ", ";
}
@@ -306,7 +320,7 @@ public function mset(AbstractModel $model, array $arr): AbstractModel {
$elements = [];
$values = [];
foreach ($arr as $key => $val) {
- $elements[] = self::getMappedModelKey($model, $key) . "=" . self::binaryPlaceholder($model, $key) . " ";
+ $elements[] = self::getMappedModelKey($model, $key) . "=" . self::columnPlaceholder($model, $key) . " ";
$values[] = $val;
}
$query .= implode(", ", $elements);
@@ -333,7 +347,7 @@ public function mset(AbstractModel $model, array $arr): AbstractModel {
* @throws Exception
*/
public function set(AbstractModel $model, string $key, $value): AbstractModel {
- $query = "UPDATE " . $this->getMappedModelTable() . " SET " . self::getMappedModelKey($model, $key) . "=" . self::binaryPlaceholder($model, $key);
+ $query = "UPDATE " . $this->getMappedModelTable() . " SET " . self::getMappedModelKey($model, $key) . "=" . self::columnPlaceholder($model, $key);
$values = [];
$query = $query . " WHERE " . $model->getPrimaryKey() . "=?";
@@ -512,7 +526,7 @@ public function massSave(array $models): bool|PDOStatement {
$query .= " (" . implode(",", $keys) . ") ";
$placeholders = [];
foreach ($origKeys as $k) {
- $placeholders[] = self::binaryPlaceholder($models[0], $k);
+ $placeholders[] = self::columnPlaceholder($models[0], $k);
}
$placeHolderStr = " (" . implode(",", $placeholders) . ")";
@@ -1192,16 +1206,17 @@ public function massSingleUpdate($matchingColumn, $updateColumn, array $updates)
/**
* @param $options
- * @return bool
+ * @return PDOStatement
* @throws Exception
*/
- public function massUpdate($options): bool {
+ public function massUpdate($options): PDOStatement {
$vals = [];
$query = $this->buildMassUpdateQuery($options, $vals);
$dbh = self::getDB();
$stmt = $dbh->prepare($query);
- return $stmt->execute($vals);
+ $stmt->execute($vals);
+ return $stmt;
}
/**
@@ -1300,4 +1315,3 @@ public function getDB(bool $test = false, array $testProperties = []): ?PDO {
}
}
}
-
diff --git a/src/dba/Factory.php b/src/dba/Factory.php
index d17b9c337..e158a7c3a 100644
--- a/src/dba/Factory.php
+++ b/src/dba/Factory.php
@@ -49,6 +49,7 @@
use Hashtopolis\dba\models\FilePretaskFactory;
use Hashtopolis\dba\models\SupertaskPretaskFactory;
use Hashtopolis\dba\models\HashlistHashlistFactory;
+use Hashtopolis\dba\models\BackgroundJobFactory;
use Hashtopolis\dba\models\_sqlx_migrationsFactory;
class Factory {
@@ -99,6 +100,7 @@ class Factory {
private static ?FilePretaskFactory $filePretaskFactory = null;
private static ?SupertaskPretaskFactory $supertaskPretaskFactory = null;
private static ?HashlistHashlistFactory $hashlistHashlistFactory = null;
+ private static ?BackgroundJobFactory $backgroundJobFactory = null;
private static ?_sqlx_migrationsFactory $_sqlx_migrationsFactory = null;
public static function getAccessGroupFactory(): AccessGroupFactory {
@@ -571,6 +573,16 @@ public static function getHashlistHashlistFactory(): HashlistHashlistFactory {
}
}
+ public static function getBackgroundJobFactory(): BackgroundJobFactory {
+ if (self::$backgroundJobFactory == null) {
+ $f = new BackgroundJobFactory();
+ self::$backgroundJobFactory = $f;
+ return $f;
+ } else {
+ return self::$backgroundJobFactory;
+ }
+ }
+
public static function get_sqlx_migrationsFactory(): _sqlx_migrationsFactory {
if (self::$_sqlx_migrationsFactory == null) {
$f = new _sqlx_migrationsFactory();
diff --git a/src/dba/models/BackgroundJob.php b/src/dba/models/BackgroundJob.php
new file mode 100644
index 000000000..53d09fdd3
--- /dev/null
+++ b/src/dba/models/BackgroundJob.php
@@ -0,0 +1,175 @@
+backgroundJobId = $backgroundJobId;
+ $this->jobType = $jobType;
+ $this->payload = $payload;
+ $this->status = $status;
+ $this->userId = $userId;
+ $this->createdAt = $createdAt;
+ $this->startedAt = $startedAt;
+ $this->finishedAt = $finishedAt;
+ $this->exitCode = $exitCode;
+ $this->resultMessage = $resultMessage;
+ }
+
+ function getKeyValueDict(): array {
+ $dict = array();
+ $dict['backgroundJobId'] = $this->backgroundJobId;
+ $dict['jobType'] = $this->jobType;
+ $dict['payload'] = $this->payload;
+ $dict['status'] = $this->status;
+ $dict['userId'] = $this->userId;
+ $dict['createdAt'] = $this->createdAt;
+ $dict['startedAt'] = $this->startedAt;
+ $dict['finishedAt'] = $this->finishedAt;
+ $dict['exitCode'] = $this->exitCode;
+ $dict['resultMessage'] = $this->resultMessage;
+
+ return $dict;
+ }
+
+ static function getFeatures(): array {
+ $dict = array();
+ $dict['backgroundJobId'] = ['read_only' => True, "type" => "int", "subtype" => "unset", "choices" => "unset", "null" => False, "pk" => True, "protected" => True, "private" => False, "alias" => "backgroundJobId", "public" => False, "dba_mapping" => False];
+ $dict['jobType'] = ['read_only' => True, "type" => "str(100)", "subtype" => "unset", "choices" => "unset", "null" => False, "pk" => False, "protected" => True, "private" => False, "alias" => "jobType", "public" => False, "dba_mapping" => False];
+ $dict['payload'] = ['read_only' => True, "type" => "json", "subtype" => "unset", "choices" => "unset", "null" => False, "pk" => False, "protected" => True, "private" => False, "alias" => "payload", "public" => False, "dba_mapping" => False];
+ $dict['status'] = ['read_only' => True, "type" => "int", "subtype" => "unset", "choices" => [-1 => "Failed", 0 => "Pending", 1 => "Running", 2 => "Done", ], "null" => False, "pk" => False, "protected" => True, "private" => False, "alias" => "status", "public" => False, "dba_mapping" => False];
+ $dict['userId'] = ['read_only' => True, "type" => "int", "subtype" => "unset", "choices" => "unset", "null" => True, "pk" => False, "protected" => True, "private" => False, "alias" => "userId", "public" => False, "dba_mapping" => False];
+ $dict['createdAt'] = ['read_only' => True, "type" => "int64", "subtype" => "unset", "choices" => "unset", "null" => False, "pk" => False, "protected" => True, "private" => False, "alias" => "createdAt", "public" => False, "dba_mapping" => False];
+ $dict['startedAt'] = ['read_only' => True, "type" => "int64", "subtype" => "unset", "choices" => "unset", "null" => True, "pk" => False, "protected" => True, "private" => False, "alias" => "startedAt", "public" => False, "dba_mapping" => False];
+ $dict['finishedAt'] = ['read_only' => True, "type" => "int64", "subtype" => "unset", "choices" => "unset", "null" => True, "pk" => False, "protected" => True, "private" => False, "alias" => "finishedAt", "public" => False, "dba_mapping" => False];
+ $dict['exitCode'] = ['read_only' => True, "type" => "int", "subtype" => "unset", "choices" => "unset", "null" => True, "pk" => False, "protected" => True, "private" => False, "alias" => "exitCode", "public" => False, "dba_mapping" => False];
+ $dict['resultMessage'] = ['read_only' => True, "type" => "str(1024)", "subtype" => "unset", "choices" => "unset", "null" => True, "pk" => False, "protected" => True, "private" => False, "alias" => "resultMessage", "public" => False, "dba_mapping" => False];
+
+ return $dict;
+ }
+
+ function getPrimaryKey(): string {
+ return "backgroundJobId";
+ }
+
+ function getPrimaryKeyValue(): ?int {
+ return $this->backgroundJobId;
+ }
+
+ function getId(): ?int {
+ return $this->backgroundJobId;
+ }
+
+ function setId($id): void {
+ $this->backgroundJobId = $id;
+ }
+
+ /**
+ * Used to serialize the data contained in the model
+ * @return array
+ */
+ public function expose(): array {
+ return get_object_vars($this);
+ }
+
+ function getJobType(): ?string {
+ return $this->jobType;
+ }
+
+ function setJobType(?string $jobType): void {
+ $this->jobType = $jobType;
+ }
+
+ function getPayload(): ?string {
+ return $this->payload;
+ }
+
+ function setPayload(?string $payload): void {
+ $this->payload = $payload;
+ }
+
+ function getStatus(): ?int {
+ return $this->status;
+ }
+
+ function setStatus(?int $status): void {
+ $this->status = $status;
+ }
+
+ function getUserId(): ?int {
+ return $this->userId;
+ }
+
+ function setUserId(?int $userId): void {
+ $this->userId = $userId;
+ }
+
+ function getCreatedAt(): ?int {
+ return $this->createdAt;
+ }
+
+ function setCreatedAt(?int $createdAt): void {
+ $this->createdAt = $createdAt;
+ }
+
+ function getStartedAt(): ?int {
+ return $this->startedAt;
+ }
+
+ function setStartedAt(?int $startedAt): void {
+ $this->startedAt = $startedAt;
+ }
+
+ function getFinishedAt(): ?int {
+ return $this->finishedAt;
+ }
+
+ function setFinishedAt(?int $finishedAt): void {
+ $this->finishedAt = $finishedAt;
+ }
+
+ function getExitCode(): ?int {
+ return $this->exitCode;
+ }
+
+ function setExitCode(?int $exitCode): void {
+ $this->exitCode = $exitCode;
+ }
+
+ function getResultMessage(): ?string {
+ return $this->resultMessage;
+ }
+
+ function setResultMessage(?string $resultMessage): void {
+ $this->resultMessage = $resultMessage;
+ }
+
+ const BACKGROUND_JOB_ID = "backgroundJobId";
+ const JOB_TYPE = "jobType";
+ const PAYLOAD = "payload";
+ const STATUS = "status";
+ const USER_ID = "userId";
+ const CREATED_AT = "createdAt";
+ const STARTED_AT = "startedAt";
+ const FINISHED_AT = "finishedAt";
+ const EXIT_CODE = "exitCode";
+ const RESULT_MESSAGE = "resultMessage";
+
+ const PERM_CREATE = "permBackgroundJobCreate";
+ const PERM_READ = "permBackgroundJobRead";
+ const PERM_UPDATE = "permBackgroundJobUpdate";
+ const PERM_DELETE = "permBackgroundJobDelete";
+}
diff --git a/src/dba/models/BackgroundJobFactory.php b/src/dba/models/BackgroundJobFactory.php
new file mode 100644
index 000000000..076d845ff
--- /dev/null
+++ b/src/dba/models/BackgroundJobFactory.php
@@ -0,0 +1,50 @@
+
+ */
+class BackgroundJobFactory extends AbstractModelFactory {
+ function getModelName(): string {
+ return "BackgroundJob";
+ }
+
+ function getModelTable(): string {
+ return "BackgroundJob";
+ }
+
+ function isMapping(): bool {
+ return False;
+ }
+
+ function isCachable(): bool {
+ return false;
+ }
+
+ function getCacheValidTime(): int {
+ return -1;
+ }
+
+ /**
+ * @return BackgroundJob
+ */
+ function getNullObject(): BackgroundJob {
+ return new BackgroundJob(-1, null, null, null, null, null, null, null, null, null);
+ }
+
+ /**
+ * @param array $dict
+ * @return BackgroundJob
+ */
+ function createObjectFromDict(array $dict): BackgroundJob {
+ $conv = [];
+ foreach ($dict as $key => $val) {
+ $conv[strtolower($key)] = $val;
+ }
+ $dict = $conv;
+ return new BackgroundJob($dict['backgroundjobid'], $dict['jobtype'], $dict['payload'], $dict['status'], $dict['userid'], $dict['createdat'], $dict['startedat'], $dict['finishedat'], $dict['exitcode'], $dict['resultmessage']);
+ }
+}
diff --git a/src/dba/models/generator.php b/src/dba/models/generator.php
index 04f1578af..57bca78a5 100644
--- a/src/dba/models/generator.php
+++ b/src/dba/models/generator.php
@@ -2,6 +2,7 @@
use Hashtopolis\inc\defines\DAgentIgnoreErrors;
use Hashtopolis\inc\defines\DAgentStatsType;
+use Hashtopolis\inc\defines\DBackgroundJobStatus;
use Hashtopolis\inc\defines\DFileDownloadStatus;
use Hashtopolis\inc\defines\DFileType;
use Hashtopolis\inc\defines\DHashcatStatus;
@@ -20,6 +21,7 @@
require_once(dirname(__FILE__) . "/../../inc/defines/DAgentIgnoreErrors.php");
require_once(dirname(__FILE__) . "/../../inc/defines/DAgentStatsType.php");
+require_once(dirname(__FILE__) . "/../../inc/defines/DBackgroundJobStatus.php");
require_once(dirname(__FILE__) . "/../../inc/defines/DFileDownloadStatus.php");
require_once(dirname(__FILE__) . "/../../inc/defines/DFileType.php");
require_once(dirname(__FILE__) . "/../../inc/defines/DHashcatStatus.php");
@@ -113,6 +115,13 @@
['key' => DHealthCheckAgentStatus::COMPLETED, 'label' => 'Completed'],
];
+$FieldBackgroundJobStatusChoices = [
+ ['key' => DBackgroundJobStatus::FAILED, 'label' => 'Failed'],
+ ['key' => DBackgroundJobStatus::PENDING, 'label' => 'Pending'],
+ ['key' => DBackgroundJobStatus::RUNNING, 'label' => 'Running'],
+ ['key' => DBackgroundJobStatus::DONE, 'label' => 'Done'],
+];
+
$FieldLogEntryIssuerChoices = [
['key' => DLogEntryIssuer::API, 'label' => 'API'],
['key' => DLogEntryIssuer::USER, 'label' => 'User'],
@@ -684,6 +693,21 @@
],
];
+$CONF['BackgroundJob'] = [
+ 'columns' => [
+ ['name' => 'backgroundJobId', 'read_only' => True, 'type' => 'int', 'protected' => True],
+ ['name' => 'jobType', 'read_only' => True, 'type' => 'str(100)', 'protected' => True],
+ ['name' => 'payload', 'read_only' => True, 'type' => 'json', 'protected' => True],
+ ['name' => 'status', 'read_only' => True, 'type' => 'int', 'protected' => True, 'choices' => $FieldBackgroundJobStatusChoices],
+ ['name' => 'userId', 'read_only' => True, 'protected' => True, 'type' => 'int', 'relation' => 'User', 'null' => True],
+ ['name' => 'createdAt', 'read_only' => True, 'type' => 'int64', 'protected' => True],
+ ['name' => 'startedAt', 'read_only' => True, 'type' => 'int64', 'protected' => True, 'null' => True],
+ ['name' => 'finishedAt', 'read_only' => True, 'type' => 'int64', 'protected' => True, 'null' => True],
+ ['name' => 'exitCode', 'read_only' => True, 'type' => 'int', 'protected' => True, 'null' => True],
+ ['name' => 'resultMessage', 'read_only' => True, 'type' => 'str(1024)', 'protected' => True, 'null' => True],
+ ],
+];
+
$CONF['_sqlx_migrations'] = [
'columns' => [
['name' => 'version', 'read_only' => True, 'type' => 'str(256)', 'protected' => True],
@@ -717,6 +741,9 @@ function getTypingType($str, $nullable = false): string {
if ($str == 'binary') {
return ($nullable ? '?' : '') . 'string';
}
+ if ($str == 'json') {
+ return ($nullable ? '?' : '') . 'string';
+ }
throw new Exception("Cannot convert type " . $str);
}
diff --git a/src/inc/Util.php b/src/inc/Util.php
index d76aaf9a1..10171278e 100755
--- a/src/inc/Util.php
+++ b/src/inc/Util.php
@@ -785,6 +785,32 @@ public static function fileLineCount(string $file): int {
}
// find out what a prettier solution for this would be, as opposed to setting the max execution time to an arbitrary two hours
ini_set('max_execution_time', '7200');
+ if (str_ends_with($file, '.gz')) {
+ // gzip files are counted on the decompressed stream. It is assumed that the archive contains exactly one file.
+ $raw = @fopen($file, 'rb');
+ if ($raw === false) {
+ return -1;
+ }
+ $magic = fread($raw, 2);
+ fclose($raw);
+ if ($magic !== "\x1f\x8b") {
+ return -1;
+ }
+ $handle = @fopen('compress.zlib://' . $file, 'rb');
+ if ($handle === false) {
+ return -1;
+ }
+ $count = 0;
+ while (($line = @fgets($handle)) !== false) {
+ $count++;
+ }
+ if (!feof($handle)) {
+ fclose($handle);
+ return -1;
+ }
+ fclose($handle);
+ return $count;
+ }
$file = new SplFileObject($file, 'r');
$file->seek(PHP_INT_MAX);
diff --git a/src/inc/apiv2/common/AbstractBaseAPI.php b/src/inc/apiv2/common/AbstractBaseAPI.php
index 4321c25c0..a853112a1 100644
--- a/src/inc/apiv2/common/AbstractBaseAPI.php
+++ b/src/inc/apiv2/common/AbstractBaseAPI.php
@@ -30,6 +30,7 @@
use Hashtopolis\dba\models\AgentError;
use Hashtopolis\dba\models\AgentStat;
use Hashtopolis\dba\models\Assignment;
+use Hashtopolis\dba\models\BackgroundJob;
use Hashtopolis\dba\models\Chunk;
use Hashtopolis\dba\ComparisonFilter;
use Hashtopolis\dba\models\Config;
@@ -346,6 +347,8 @@ protected static function getModelFactory(string $model): AbstractModelFactory {
return Factory::getAgentErrorFactory();
case AgentStat::class:
return Factory::getAgentStatFactory();
+ case BackgroundJob::class:
+ return Factory::getBackgroundJobFactory();
case Assignment::class:
return Factory::getAssignmentFactory();
case Chunk::class:
@@ -632,7 +635,9 @@ protected static function getExpandPermissions(string $expand): array {
HealthCheck::PERM_CREATE, HealthCheck::PERM_READ, HealthCheck::PERM_UPDATE, HealthCheck::PERM_DELETE,
HealthCheckAgent::PERM_CREATE, HealthCheckAgent::PERM_READ, HealthCheckAgent::PERM_UPDATE, HealthCheckAgent::PERM_DELETE,
// src/inc/defines/hashlists.php
- HashType::PERM_CREATE, HashType::PERM_READ, HashType::PERM_UPDATE, HashType::PERM_DELETE
+ HashType::PERM_CREATE, HashType::PERM_READ, HashType::PERM_UPDATE, HashType::PERM_DELETE,
+ // src/dba/models/BackgroundJob.php
+ BackgroundJob::PERM_CREATE, BackgroundJob::PERM_READ, BackgroundJob::PERM_UPDATE, BackgroundJob::PERM_DELETE
),
DAccessControl::USER_CONFIG_ACCESS => array(User::PERM_CREATE, User::PERM_READ, User::PERM_UPDATE, User::PERM_DELETE, RightGroup::PERM_CREATE, RightGroup::PERM_READ, RightGroup::PERM_UPDATE, RightGroup::PERM_DELETE),
@@ -664,6 +669,12 @@ protected static function db2json(array $feature, mixed $val): mixed {
$obj = (object)[];
}
}
+ elseif ($feature['type'] == 'json') {
+ $obj = json_decode($val, true, 512, JSON_OBJECT_AS_ARRAY);
+ if (empty($obj)) {
+ $obj = (object)[];
+ }
+ }
elseif ($feature['type'] == 'array' && $feature['subtype'] == 'int') {
$obj = array_map('intval', preg_split("/,/", $val, -1, PREG_SPLIT_NO_EMPTY));
}
@@ -699,6 +710,9 @@ protected static function json2db(array $feature, mixed $obj): ?string {
elseif ($feature['type'] == 'dict' && $feature['subtype'] == 'bool') {
$val = serialize($obj);
}
+ elseif ($feature['type'] == 'json') {
+ $val = json_encode($obj);
+ }
else {
$val = strval($obj);
}
@@ -1083,6 +1097,12 @@ protected function validateData(array $data, array $features): void {
throw new HttpError("Key '$key' dict contains non-boolean values");
}
}
+ // Json
+ }
+ elseif (str_starts_with($features[$key]['type'], 'json')) {
+ if (!is_array($value)) {
+ throw new HttpError("Key '$key' is not of type json (object)");
+ }
}
else {
throw new HttpError("Typemapping error for key '$key' ");
diff --git a/src/inc/apiv2/common/ApiRegistry.php b/src/inc/apiv2/common/ApiRegistry.php
index 1611028e0..6b9efc7f4 100644
--- a/src/inc/apiv2/common/ApiRegistry.php
+++ b/src/inc/apiv2/common/ApiRegistry.php
@@ -41,6 +41,7 @@
use Hashtopolis\inc\apiv2\model\AgentErrorAPI;
use Hashtopolis\inc\apiv2\model\AgentStatAPI;
use Hashtopolis\inc\apiv2\model\ApiTokenAPI;
+use Hashtopolis\inc\apiv2\model\BackgroundJobAPI;
use Hashtopolis\inc\apiv2\model\ChunkAPI;
use Hashtopolis\inc\apiv2\model\ConfigAPI;
use Hashtopolis\inc\apiv2\model\ConfigSectionAPI;
@@ -82,6 +83,7 @@ final class ApiRegistry {
AgentErrorAPI::class,
AgentStatAPI::class,
ApiTokenAPI::class,
+ BackgroundJobAPI::class,
ChunkAPI::class,
ConfigAPI::class,
ConfigSectionAPI::class,
diff --git a/src/inc/apiv2/helper/RecountFileLinesHelperAPI.php b/src/inc/apiv2/helper/RecountFileLinesHelperAPI.php
index bd49d4ace..c818a486a 100644
--- a/src/inc/apiv2/helper/RecountFileLinesHelperAPI.php
+++ b/src/inc/apiv2/helper/RecountFileLinesHelperAPI.php
@@ -3,8 +3,10 @@
namespace Hashtopolis\inc\apiv2\helper;
use Hashtopolis\dba\AbstractModel;
+use Hashtopolis\inc\utils\BackgroundJobUtils;
use Hashtopolis\inc\utils\FileUtils;
use Hashtopolis\dba\models\File;
+use Hashtopolis\inc\defines\DBackgroundJobType;
use Hashtopolis\inc\apiv2\common\AbstractHelperAPI;
use Hashtopolis\inc\HTException;
use Psr\Container\ContainerExceptionInterface;
@@ -37,19 +39,18 @@ public static function getResponse(): string {
}
/**
- * Endpoint to recount files for when there is size mismatch
+ * Endpoint to enqueue the recounting of files for when there is size mismatch. The line count is
+ * calculated asynchronously by the background job runner.
* @param $data
* @return AbstractModel|array|null
* @throws HTException
- * @throws ContainerExceptionInterface
- * @throws NotFoundExceptionInterface
*/
public function actionPost($data): AbstractModel|array|null {
- // first retrieve the file, as fileCountLines does not check any permissions, therefore to be sure call getFile() first, even if it is not required technically
+ // first retrieve the file, as this checks the access permissions, which enqueueing the job does not do
FileUtils::getFile($data[File::FILE_ID], $this->getCurrentUser());
- FileUtils::fileCountLines($data[File::FILE_ID]);
-
+ BackgroundJobUtils::enqueue(DBackgroundJobType::RECOUNT_FILE, [File::FILE_ID => $data[File::FILE_ID]], $this->getCurrentUser());
+
/* Return the File itself so it comes back as a resource object under data.
Only the helper's own permission (File update) is checked; the serializer
no longer demands the model route's create permission. */
diff --git a/src/inc/apiv2/model/BackgroundJobAPI.php b/src/inc/apiv2/model/BackgroundJobAPI.php
new file mode 100644
index 000000000..5ac2d803e
--- /dev/null
+++ b/src/inc/apiv2/model/BackgroundJobAPI.php
@@ -0,0 +1,61 @@
+
+ */
+class BackgroundJobAPI extends AbstractModelAPI {
+ public static function getBaseUri(): string {
+ return "/api/v2/ui/backgroundJobs";
+ }
+
+ public static function getAvailableMethods(): array {
+ return ['GET', 'DELETE'];
+ }
+
+ public static function getDBAclass(): string {
+ return BackgroundJob::class;
+ }
+
+ public static function getToOneRelationships(): array {
+ return [
+ 'user' => [
+ 'key' => BackgroundJob::USER_ID,
+
+ 'relationType' => User::class,
+ 'relationKey' => User::USER_ID,
+ ],
+ ];
+ }
+
+ /**
+ * @throws HttpError
+ */
+ protected function createObject(array $data): int {
+ throw new HttpError("Background jobs cannot be created via API");
+ }
+
+ /**
+ * @throws HttpError
+ */
+ public function updateObject(int $objectId, array $data): void {
+ throw new HttpError("Background jobs cannot be updated via API");
+ }
+
+ /**
+ * @param BackgroundJob $object
+ * @throws HttpConflict
+ */
+ protected function deleteObject(AbstractModel $object): void {
+ BackgroundJobUtils::deleteJob($object);
+ }
+}
diff --git a/src/inc/apiv2/openapi/FeatureTypeMapper.php b/src/inc/apiv2/openapi/FeatureTypeMapper.php
index a1b45cab4..c3c034347 100644
--- a/src/inc/apiv2/openapi/FeatureTypeMapper.php
+++ b/src/inc/apiv2/openapi/FeatureTypeMapper.php
@@ -33,6 +33,9 @@ public function typeLookup($feature): array {
$sub_type = $this->typeLookup(['type' => $feature['subtype'], 'choices' => 'unset'])['type'];
}
}
+ elseif ($feature['type'] == 'json') {
+ $type = "object";
+ }
elseif ($feature['type'] == 'array') {
$type = "array";
$sub_type = "integer"; //TODO: subtype is hardcoded because we only have int arrays
diff --git a/src/inc/apiv2/openapi/RouteIntrospector.php b/src/inc/apiv2/openapi/RouteIntrospector.php
index 3a9f2189b..d710cee7e 100644
--- a/src/inc/apiv2/openapi/RouteIntrospector.php
+++ b/src/inc/apiv2/openapi/RouteIntrospector.php
@@ -2,6 +2,8 @@
namespace Hashtopolis\inc\apiv2\openapi;
+use ReflectionException;
+use ReflectionObject;
use Slim\App;
/**
@@ -13,13 +15,14 @@
class RouteIntrospector {
/**
* @return list
+ * @throws ReflectionException
*/
public function introspect(App $app): array {
$targets = [];
$routes = $app->getRouteCollector()->getRoutes();
foreach ($routes as $route) {
/* Quirk to receive className, since it is hidden in a protected variable */
- $reflectionOfRoute = new \ReflectionObject($route);
+ $reflectionOfRoute = new ReflectionObject($route);
$protectedCallable = $reflectionOfRoute->getProperty('callable');
$reflectionCallable = ($protectedCallable->getValue($route));
diff --git a/src/inc/apiv2/openapi/SpecSanitizer.php b/src/inc/apiv2/openapi/SpecSanitizer.php
index ac2fff735..92885d173 100644
--- a/src/inc/apiv2/openapi/SpecSanitizer.php
+++ b/src/inc/apiv2/openapi/SpecSanitizer.php
@@ -2,6 +2,8 @@
namespace Hashtopolis\inc\apiv2\openapi;
+use stdClass as stdClassAlias;
+
/**
* Post-processes the raw generated spec for strict OpenAPI 3.1.0 compliance.
*/
@@ -321,7 +323,7 @@ private function recursiveFixValues(array $data, array $renameMap): array {
if ($key === 'properties' && is_array($value)) {
if (empty($value)) {
// Empty array -> stdClass so json_encode outputs {} not []
- $value = new \stdClass();
+ $value = new stdClassAlias();
continue;
}
if (isset($value[0])) {
diff --git a/src/inc/cron.php b/src/inc/cron.php
new file mode 100644
index 000000000..70a23f0a7
--- /dev/null
+++ b/src/inc/cron.php
@@ -0,0 +1,9 @@
+checkPermission(DFileAction::COUNT_FILE_LINES_PERM);
- FileUtils::fileCountLines($_POST['file']);
- UI::addMessage(UI::SUCCESS, "Line count has been successfully calculated!");
+ BackgroundJobUtils::enqueue(DBackgroundJobType::RECOUNT_FILE, ['fileId' => intval($_POST['file'])], AccessControl::getInstance()->getUser());
+ UI::addMessage(UI::SUCCESS, "Line count calculation was enqueued as background job!");
break;
default:
UI::addMessage(UI::ERROR, "Invalid action!");
diff --git a/src/inc/jobs/BackgroundJobHandler.php b/src/inc/jobs/BackgroundJobHandler.php
new file mode 100644
index 000000000..b30e364b6
--- /dev/null
+++ b/src/inc/jobs/BackgroundJobHandler.php
@@ -0,0 +1,33 @@
+ payload key to field definition
+ */
+ public static function getPayloadDefinition(): array;
+
+ /**
+ * @return int maximum runtime in seconds, after which a running job is considered stale
+ */
+ public function getMaxRuntime(): int;
+
+ /**
+ * @param BackgroundJob $job claimed job which is about to be executed
+ * @param array $payload decoded JSON payload of the job
+ * @return BackgroundJobResult exit code 0 marks success, anything else marks failure
+ */
+ public function execute(BackgroundJob $job, array $payload): BackgroundJobResult;
+}
diff --git a/src/inc/jobs/BackgroundJobPayloadValidator.php b/src/inc/jobs/BackgroundJobPayloadValidator.php
new file mode 100644
index 000000000..7a8111a02
--- /dev/null
+++ b/src/inc/jobs/BackgroundJobPayloadValidator.php
@@ -0,0 +1,30 @@
+ $definition payload key to field definition
+ * @throws HTException if the payload violates the definition
+ */
+ public static function validate(array $payload, array $definition): void {
+ foreach ($definition as $key => $field) {
+ if (!array_key_exists($key, $payload)) {
+ if ($field->required) {
+ throw new HTException("Missing required payload key '$key'!");
+ }
+ continue;
+ }
+ if (!$field->checkType($payload[$key])) {
+ throw new HTException("Invalid type for payload key '$key', expected " . $field->type . "!");
+ }
+ }
+ }
+}
diff --git a/src/inc/jobs/BackgroundJobRegistry.php b/src/inc/jobs/BackgroundJobRegistry.php
new file mode 100644
index 000000000..d68743bb2
--- /dev/null
+++ b/src/inc/jobs/BackgroundJobRegistry.php
@@ -0,0 +1,32 @@
+> */
+ private const HANDLERS = [
+ DBackgroundJobType::RECOUNT_FILE => RecountFileJob::class,
+ ];
+
+ /**
+ * @return string[] list of all registered job type identifiers
+ */
+ public static function getRegisteredTypes(): array {
+ return array_keys(self::HANDLERS);
+ }
+
+ public static function getHandlerClass(string $jobType): ?string {
+ return self::HANDLERS[$jobType] ?? null;
+ }
+
+ public static function getHandler(string $jobType): ?BackgroundJobHandler {
+ $class = self::getHandlerClass($jobType);
+ if ($class === null) {
+ return null;
+ }
+ return new $class();
+ }
+}
diff --git a/src/inc/jobs/BackgroundJobResult.php b/src/inc/jobs/BackgroundJobResult.php
new file mode 100644
index 000000000..eba993b7f
--- /dev/null
+++ b/src/inc/jobs/BackgroundJobResult.php
@@ -0,0 +1,21 @@
+exitCode = $exitCode;
+ $this->message = $message;
+ }
+
+ public function getExitCode(): int {
+ return $this->exitCode;
+ }
+
+ public function getMessage(): ?string {
+ return $this->message;
+ }
+}
diff --git a/src/inc/jobs/BackgroundJobRunner.php b/src/inc/jobs/BackgroundJobRunner.php
new file mode 100644
index 000000000..2809808d4
--- /dev/null
+++ b/src/inc/jobs/BackgroundJobRunner.php
@@ -0,0 +1,131 @@
+filter([
+ Factory::FILTER => new QueryFilter(BackgroundJob::STATUS, DBackgroundJobStatus::PENDING, "="),
+ ], true);
+ if ($job === null) {
+ break;
+ }
+ if (!self::claim($job)) {
+ continue;
+ }
+ self::executeJob($job);
+ }
+ }
+ finally {
+ LockUtils::release(Lock::BACKGROUND_JOBS);
+ }
+ }
+
+ /**
+ * Atomically transitions a pending job into the running state. Returns false if the
+ * job was claimed by another runner instance in the meantime.
+ * @throws Exception
+ */
+ private static function claim(BackgroundJob $job): bool {
+ $factory = Factory::getBackgroundJobFactory();
+ $stmt = $factory->massUpdate([
+ Factory::UPDATE => [
+ new UpdateSet(BackgroundJob::STATUS, DBackgroundJobStatus::RUNNING),
+ new UpdateSet(BackgroundJob::STARTED_AT, time()),
+ ],
+ Factory::FILTER => [
+ new QueryFilter(BackgroundJob::BACKGROUND_JOB_ID, $job->getId(), "="),
+ new QueryFilter(BackgroundJob::STATUS, DBackgroundJobStatus::PENDING, "="),
+ ],
+ ]);
+ return $stmt->rowCount() > 0;
+ }
+
+ /**
+ * Marks running jobs as failed which were left behind by a crashed or killed runner,
+ * which exceeded their maximum runtime, or which no longer have a registered handler.
+ * @throws Exception
+ */
+ private static function recoverStale(): void {
+ $factory = Factory::getBackgroundJobFactory();
+ $running = $factory->filter([Factory::FILTER => new QueryFilter(BackgroundJob::STATUS, DBackgroundJobStatus::RUNNING, "=")]);
+ foreach ($running as $job) {
+ $handlerClass = BackgroundJobRegistry::getHandlerClass($job->getJobType());
+ $handler = ($handlerClass === null) ? null : new $handlerClass();
+ $stale = ($job->getStartedAt() === null) || ($handler === null) || (time() - $job->getStartedAt() > $handler->getMaxRuntime());
+ if ($stale) {
+ $reason = ($handler === null) ? "No handler registered for job type '" . $job->getJobType() . "'!" : "Job timed out or runner died!";
+ $factory->mset($job, [
+ BackgroundJob::STATUS => DBackgroundJobStatus::FAILED,
+ BackgroundJob::EXIT_CODE => -1,
+ BackgroundJob::RESULT_MESSAGE => $reason,
+ BackgroundJob::FINISHED_AT => time(),
+ ]);
+ DServerLog::log(DServerLog::WARNING, "Background job " . $job->getId() . " (" . $job->getJobType() . ") was stale and got marked as failed: " . $reason, [$job]);
+ }
+ }
+ }
+
+ /**
+ * Executes a single claimed job and records its result, exit code and message. Any
+ * uncaught throwable is caught and recorded as a failed execution.
+ * @throws Exception
+ */
+ private static function executeJob(BackgroundJob $job): void {
+ $factory = Factory::getBackgroundJobFactory();
+ DServerLog::log(DServerLog::INFO, "Background job " . $job->getId() . " (" . $job->getJobType() . ") started", [$job]);
+ try {
+ $handler = BackgroundJobRegistry::getHandler($job->getJobType());
+ if ($handler === null) {
+ throw new HTException("No handler registered for job type '" . $job->getJobType() . "'!");
+ }
+ $payload = json_decode($job->getPayload() ?? "{}", true, 512, JSON_THROW_ON_ERROR);
+ if (!is_array($payload)) {
+ throw new HTException("Invalid job payload!");
+ }
+ BackgroundJobPayloadValidator::validate($payload, $handler::getPayloadDefinition());
+ $result = $handler->execute($job, $payload);
+ $factory->mset($job, [
+ BackgroundJob::STATUS => ($result->getExitCode() === 0) ? DBackgroundJobStatus::DONE : DBackgroundJobStatus::FAILED,
+ BackgroundJob::EXIT_CODE => $result->getExitCode(),
+ BackgroundJob::RESULT_MESSAGE => $result->getMessage(),
+ BackgroundJob::FINISHED_AT => time(),
+ ]);
+ DServerLog::log(DServerLog::INFO, "Background job " . $job->getId() . " (" . $job->getJobType() . ") finished with exit code " . $result->getExitCode());
+ }
+ catch (Throwable $t) {
+ $message = substr($t->getMessage(), 0, 1024);
+ $factory->mset($job, [
+ BackgroundJob::STATUS => DBackgroundJobStatus::FAILED,
+ BackgroundJob::EXIT_CODE => 255,
+ BackgroundJob::RESULT_MESSAGE => $message,
+ BackgroundJob::FINISHED_AT => time(),
+ ]);
+ DServerLog::log(DServerLog::ERROR, "Background job " . $job->getId() . " (" . $job->getJobType() . ") failed: " . $message);
+ }
+ }
+}
diff --git a/src/inc/jobs/handlers/RecountFileJob.php b/src/inc/jobs/handlers/RecountFileJob.php
new file mode 100644
index 000000000..9046580c3
--- /dev/null
+++ b/src/inc/jobs/handlers/RecountFileJob.php
@@ -0,0 +1,42 @@
+ new JobPayloadField(JobPayloadField::TYPE_INT),
+ ];
+ }
+
+ public function getMaxRuntime(): int {
+ return 7200;
+ }
+
+ /**
+ * @throws Exception
+ */
+ public function execute(BackgroundJob $job, array $payload): BackgroundJobResult {
+ try {
+ $count = FileUtils::fileCountLines($payload[File::FILE_ID]);
+ }
+ catch (HTException $e) {
+ return new BackgroundJobResult(-1, $e->getMessage());
+ }
+ return new BackgroundJobResult(0, "Recounted $count lines.");
+ }
+}
diff --git a/src/inc/jobs/payload/JobPayloadField.php b/src/inc/jobs/payload/JobPayloadField.php
new file mode 100644
index 000000000..18d70006d
--- /dev/null
+++ b/src/inc/jobs/payload/JobPayloadField.php
@@ -0,0 +1,27 @@
+type) {
+ self::TYPE_INT => is_int($value),
+ self::TYPE_STRING => is_string($value),
+ self::TYPE_BOOL => is_bool($value),
+ self::TYPE_FLOAT => is_float($value),
+ self::TYPE_ARRAY => is_array($value),
+ default => false,
+ };
+ }
+}
diff --git a/src/inc/utils/BackgroundJobUtils.php b/src/inc/utils/BackgroundJobUtils.php
new file mode 100644
index 000000000..02f5af8d6
--- /dev/null
+++ b/src/inc/utils/BackgroundJobUtils.php
@@ -0,0 +1,70 @@
+getId(), time(), null, null, null, null);
+ $job = Factory::getBackgroundJobFactory()->save($job);
+ if ($job === null) {
+ throw new HTException("Could not enqueue background job!");
+ }
+ DServerLog::log(DServerLog::INFO, "Background job " . $job->getId() . " (" . $jobType . ") enqueued", [$job]);
+ return $job;
+ }
+
+ /**
+ * @throws HTException
+ */
+ public static function getJob(int $jobId): BackgroundJob {
+ $job = Factory::getBackgroundJobFactory()->get($jobId);
+ if ($job === null) {
+ throw new HTException("No such background job!");
+ }
+ return $job;
+ }
+
+ /**
+ * @throws HttpConflict
+ */
+ public static function deleteJob(BackgroundJob $job): void {
+ $stmt = Factory::getBackgroundJobFactory()->massDeletion([
+ Factory::FILTER => [
+ new QueryFilter(BackgroundJob::BACKGROUND_JOB_ID, $job->getId(), "="),
+ new QueryFilter(BackgroundJob::STATUS, DBackgroundJobStatus::RUNNING, "<>"),
+ ],
+ ]);
+ if ($stmt->rowCount() === 0) {
+ throw new HttpConflict("Background job is currently running and cannot be deleted!");
+ }
+ }
+}
diff --git a/src/inc/utils/FileUtils.php b/src/inc/utils/FileUtils.php
index 492713c75..35ed0e630 100644
--- a/src/inc/utils/FileUtils.php
+++ b/src/inc/utils/FileUtils.php
@@ -352,12 +352,16 @@ public static function getFile(int $fileId, User $user): File {
}
/**
- * @param $fileId
+ * @param int $fileId
+ * @return int the recounted number of lines of the file
* @throws HTException
* @throws Exception
*/
- public static function fileCountLines($fileId): void {
+ public static function fileCountLines(int $fileId): int {
$file = Factory::getFileFactory()->get($fileId);
+ if ($file === null) {
+ throw new HTException("No such file!");
+ }
$fileName = $file->getFilename();
$filePath = Factory::getStoredValueFactory()->get(DDirectories::FILES)->getVal() . "/" . $fileName;
if (!file_exists($filePath)) {
@@ -373,8 +377,7 @@ public static function fileCountLines($fileId): void {
if ($count == -1) {
throw new HTException("Could not determine line count.");
}
- else {
- Factory::getFileFactory()->set($file, File::LINE_COUNT, $count);
- }
+ Factory::getFileFactory()->set($file, File::LINE_COUNT, $count);
+ return $count;
}
}
diff --git a/src/inc/utils/Lock.php b/src/inc/utils/Lock.php
index d677a61dc..9c2f071b3 100644
--- a/src/inc/utils/Lock.php
+++ b/src/inc/utils/Lock.php
@@ -5,8 +5,9 @@
use Exception;
class Lock {
- const CHUNKING = "chunking.lock";
- const LOG = "log.lock";
+ const CHUNKING = "chunking.lock";
+ const LOG = "log.lock";
+ const BACKGROUND_JOBS = "background-jobs.lock";
private string $lockFile;
private $lock;
@@ -37,6 +38,14 @@ public function getLock(): void {
}
}
+ /**
+ * @return bool true if the lock was acquired, false if it is currently held elsewhere
+ */
+ public function tryGetLock(): bool {
+ $ret = flock($this->lock, LOCK_EX | LOCK_NB);
+ return $ret === true;
+ }
+
/**
* @throws Exception
*/
diff --git a/src/inc/utils/LockUtils.php b/src/inc/utils/LockUtils.php
index d01e5a91d..7bf02683f 100644
--- a/src/inc/utils/LockUtils.php
+++ b/src/inc/utils/LockUtils.php
@@ -30,6 +30,28 @@ public static function get(string $lockFile): void {
}
}
+ /**
+ * Tries to acquire the given lock without blocking.
+ *
+ * @param string $lockFile
+ * @return bool true if the lock was acquired, false if it is currently held elsewhere
+ */
+ public static function tryGet(string $lockFile): bool {
+ if (isset(self::$locks[$lockFile])) {
+ $lock = self::$locks[$lockFile];
+ }
+ else {
+ try {
+ $lock = new Lock($lockFile);
+ }
+ catch (Exception $e) {
+ return false;
+ }
+ self::$locks[$lockFile] = $lock;
+ }
+ return $lock->tryGetLock();
+ }
+
/**
* @param string $lockFile
*/
diff --git a/src/inc/utils/UserUtils.php b/src/inc/utils/UserUtils.php
index 6d11bd458..f3df0f8b4 100644
--- a/src/inc/utils/UserUtils.php
+++ b/src/inc/utils/UserUtils.php
@@ -9,6 +9,7 @@
use Hashtopolis\dba\models\User;
use Hashtopolis\dba\QueryFilter;
use Hashtopolis\dba\models\AccessGroupUser;
+use Hashtopolis\dba\models\BackgroundJob;
use Hashtopolis\dba\models\Session;
use Hashtopolis\dba\models\NotificationSetting;
use Hashtopolis\dba\models\Agent;
@@ -75,6 +76,9 @@ public static function deleteUser(int $userId, User $adminUser): void {
// Revoke all of the API keys of the user
Factory::getJwtApiKeyFactory()->massUpdate([Factory::FILTER => $qF, Factory::UPDATE => [$uS1, $uS2]]);
+ $qF = new QueryFilter(BackgroundJob::USER_ID, $user->getId(), "=");
+ $uS = new UpdateSet(BackgroundJob::USER_ID, null);
+ Factory::getBackgroundJobFactory()->massUpdate([Factory::FILTER => $qF, Factory::UPDATE => $uS]);
Factory::getUserFactory()->delete($user);
}
diff --git a/src/migrations/mysql/20260917160500_background-jobs.sql b/src/migrations/mysql/20260917160500_background-jobs.sql
new file mode 100644
index 000000000..9976196c2
--- /dev/null
+++ b/src/migrations/mysql/20260917160500_background-jobs.sql
@@ -0,0 +1,16 @@
+-- Create BackgroundJob table for the cron-based background job queue.
+CREATE TABLE `BackgroundJob` (
+ `backgroundJobId` int NOT NULL AUTO_INCREMENT,
+ `jobType` varchar(100) NOT NULL,
+ `payload` json NOT NULL,
+ `status` int NOT NULL DEFAULT 0,
+ `userId` int DEFAULT NULL,
+ `createdAt` bigint NOT NULL,
+ `startedAt` bigint DEFAULT NULL,
+ `finishedAt` bigint DEFAULT NULL,
+ `exitCode` int DEFAULT NULL,
+ `resultMessage` varchar(1024) DEFAULT NULL,
+ PRIMARY KEY (`backgroundJobId`),
+ KEY `status` (`status`),
+ CONSTRAINT `BackgroundJob_ibfk_1` FOREIGN KEY (`userId`) REFERENCES `htp_User` (`userId`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
diff --git a/src/migrations/postgres/20260917160500_background-jobs.sql b/src/migrations/postgres/20260917160500_background-jobs.sql
new file mode 100644
index 000000000..a653afb36
--- /dev/null
+++ b/src/migrations/postgres/20260917160500_background-jobs.sql
@@ -0,0 +1,18 @@
+-- Create backgroundjob table for the cron-based background job queue.
+CREATE TABLE backgroundjob (
+ backgroundjobid integer NOT NULL,
+ jobtype varchar(100) NOT NULL,
+ payload json NOT NULL,
+ status integer DEFAULT 0 NOT NULL,
+ userid integer,
+ createdat bigint NOT NULL,
+ startedat bigint,
+ finishedat bigint,
+ exitcode integer,
+ resultmessage varchar(1024)
+);
+CREATE SEQUENCE backgroundjob_backgroundjobid_seq AS integer START WITH 1 INCREMENT BY 1;
+ALTER TABLE ONLY backgroundjob ALTER COLUMN backgroundjobid SET DEFAULT nextval('backgroundjob_backgroundjobid_seq'::regclass);
+ALTER TABLE ONLY backgroundjob ADD CONSTRAINT backgroundjob_pkey PRIMARY KEY (backgroundjobid);
+ALTER TABLE ONLY backgroundjob ADD CONSTRAINT backgroundjob_ibfk_1 FOREIGN KEY (userid) REFERENCES htp_user(userid);
+CREATE INDEX backgroundjob_status_idx ON backgroundjob USING btree (status);