Skip to content

Commit f2a492a

Browse files
vegarstivegarsti
committed
feat(habitat): filter terminal failures
1 parent a347eb5 commit f2a492a

4 files changed

Lines changed: 323 additions & 4 deletions

File tree

‎habitat/internal/server/handlers.go‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,7 @@ func (s *Server) handleTasks(w http.ResponseWriter, r *http.Request) {
310310
defer cancel()
311311

312312
statusFilter, statusValid := normalizeTaskStatusFilter(queryValues.Get("status"))
313+
failedRunScope, failedRunScopeValid := normalizeFailedRunScope(queryValues.Get("failedRunScope"))
313314
queueFilter := strings.TrimSpace(queryValues.Get("queue"))
314315
taskNameFilter := strings.TrimSpace(queryValues.Get("taskName"))
315316
taskIDFilter := strings.TrimSpace(queryValues.Get("taskId"))
@@ -336,7 +337,7 @@ func (s *Server) handleTasks(w http.ResponseWriter, r *http.Request) {
336337
return
337338
}
338339

339-
if !statusValid {
340+
if !statusValid || !failedRunScopeValid {
340341
writeJSON(w, http.StatusOK, emptyTaskListResponse(page, perPage, queueNames))
341342
return
342343
}
@@ -396,6 +397,7 @@ func (s *Server) handleTasks(w http.ResponseWriter, r *http.Request) {
396397
ctx,
397398
queueName,
398399
statusFilter,
400+
failedRunScope,
399401
taskNameFilter,
400402
taskIDFilter,
401403
limitPerQueue,
@@ -609,6 +611,25 @@ func allTaskStatuses() []string {
609611
return []string{"pending", "running", "sleeping", "completed", "failed", "cancelled"}
610612
}
611613

614+
const (
615+
failedRunScopeAll = "all"
616+
failedRunScopeTerminal = "terminal"
617+
)
618+
619+
func normalizeFailedRunScope(value string) (string, bool) {
620+
scope := strings.ToLower(strings.TrimSpace(value))
621+
if scope == "" {
622+
return failedRunScopeAll, true
623+
}
624+
625+
switch scope {
626+
case failedRunScopeAll, failedRunScopeTerminal:
627+
return scope, true
628+
default:
629+
return failedRunScopeAll, false
630+
}
631+
}
632+
612633
func normalizeTaskStatusFilter(value string) (string, bool) {
613634
status := strings.ToLower(strings.TrimSpace(value))
614635
if status == "" {
@@ -645,6 +666,7 @@ func (s *Server) fetchQueueTaskCandidates(
645666
ctx context.Context,
646667
queueName string,
647668
statusFilter string,
669+
failedRunScope string,
648670
taskNameFilter string,
649671
taskIDFilter string,
650672
limit int,
@@ -687,6 +709,10 @@ func (s *Server) fetchQueueTaskCandidates(
687709
params = append(params, statusFilter)
688710
clauses = append(clauses, fmt.Sprintf("r.state = $%d", len(params)))
689711
}
712+
if statusFilter == "failed" && failedRunScope == failedRunScopeTerminal {
713+
clauses = append(clauses, "t.state = 'failed'")
714+
clauses = append(clauses, "r.run_id = t.last_attempt_run")
715+
}
690716
if taskNameFilter != "" {
691717
params = append(params, taskNameFilter)
692718
clauses = append(clauses, fmt.Sprintf("t.task_name = $%d", len(params)))

‎habitat/internal/server/handlers_tasks_test.go‎

Lines changed: 201 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,6 +184,42 @@ func expectQueueTasksQuery(queueName string, limit int64) func(string, []driver.
184184
}
185185
}
186186

187+
func expectFailedQueueTasksQuery(queueName string, limit int64, terminal bool) func(string, []driver.NamedValue) error {
188+
rTable := fmt.Sprintf(`FROM absurd.%q r`, "r_"+queueName)
189+
tTable := fmt.Sprintf(`JOIN absurd.%q t ON t.task_id = r.task_id`, "t_"+queueName)
190+
return func(query string, args []driver.NamedValue) error {
191+
for _, required := range []string{
192+
rTable,
193+
tTable,
194+
"r.state = $1",
195+
"ORDER BY r.run_id DESC",
196+
} {
197+
if !strings.Contains(query, required) {
198+
return fmt.Errorf("query %q missing %q", query, required)
199+
}
200+
}
201+
for _, terminalClause := range []string{"t.state = 'failed'", "r.run_id = t.last_attempt_run"} {
202+
contains := strings.Contains(query, terminalClause)
203+
if terminal && !contains {
204+
return fmt.Errorf("query %q missing %q", query, terminalClause)
205+
}
206+
if !terminal && contains {
207+
return fmt.Errorf("query %q unexpectedly contains %q", query, terminalClause)
208+
}
209+
}
210+
if len(args) != 2 {
211+
return fmt.Errorf("expected 2 args, got %d", len(args))
212+
}
213+
if args[0].Value != "failed" {
214+
return fmt.Errorf("status arg = %#v, want failed", args[0].Value)
215+
}
216+
if args[1].Value != limit {
217+
return fmt.Errorf("limit arg = %#v, want %#v", args[1].Value, limit)
218+
}
219+
return nil
220+
}
221+
}
222+
187223
func expectRecentTaskNamesQuery(queueName string, limit int64) func(string, []driver.NamedValue) error {
188224
rTable := fmt.Sprintf(`FROM absurd.%q`, "r_"+queueName)
189225
tTable := fmt.Sprintf(`JOIN absurd.%q t ON t.task_id = r.task_id`, "t_"+queueName)
@@ -352,6 +388,171 @@ func TestHandleTasksFailsFastOnQueueQueryDeadline(t *testing.T) {
352388
}
353389
}
354390

391+
func TestHandleTasksTerminalFailedScopeFiltersLastFailedRun(t *testing.T) {
392+
now := time.Now().UTC()
393+
taskID := uuid.NewString()
394+
runID := uuid.NewString()
395+
396+
db := newScriptedDB(t, []scriptedQuery{
397+
{
398+
match: expectContains(`SELECT queue_name FROM absurd.queues ORDER BY queue_name`),
399+
columns: []string{"queue_name"},
400+
rows: [][]driver.Value{{"alpha"}},
401+
},
402+
{
403+
match: expectRecentTaskNamesQuery("alpha", 5000),
404+
columns: []string{"task_name"},
405+
rows: [][]driver.Value{{"process-webhook"}},
406+
},
407+
{
408+
match: expectFailedQueueTasksQuery("alpha", 27, true),
409+
columns: []string{
410+
"task_id",
411+
"run_id",
412+
"queue_name",
413+
"task_name",
414+
"state",
415+
"attempt",
416+
"max_attempts",
417+
"created_at",
418+
"updated_at",
419+
"completed_at",
420+
"claimed_by",
421+
"params",
422+
},
423+
rows: [][]driver.Value{
424+
{
425+
taskID,
426+
runID,
427+
"alpha",
428+
"process-webhook",
429+
"failed",
430+
int64(3),
431+
int64(3),
432+
now,
433+
now,
434+
nil,
435+
nil,
436+
nil,
437+
},
438+
},
439+
},
440+
})
441+
442+
srv := &Server{db: db}
443+
444+
req := httptest.NewRequest(http.MethodGet, "/api/tasks?status=failed&failedRunScope=terminal&page=1&perPage=25", nil)
445+
resp := httptest.NewRecorder()
446+
447+
srv.handleTasks(resp, req)
448+
449+
if resp.Code != http.StatusOK {
450+
t.Fatalf("status = %d, want %d (body=%q)", resp.Code, http.StatusOK, resp.Body.String())
451+
}
452+
453+
body := resp.Body.String()
454+
if !strings.Contains(body, runID) {
455+
t.Fatalf("expected terminal failed run in response body, got %q", body)
456+
}
457+
if !strings.Contains(body, `"total":1`) {
458+
t.Fatalf("expected total=1 in response body, got %q", body)
459+
}
460+
}
461+
462+
func TestHandleTasksAllFailedScopeKeepsRetriedFailures(t *testing.T) {
463+
now := time.Now().UTC()
464+
taskID := uuid.NewString()
465+
runID := uuid.NewString()
466+
467+
db := newScriptedDB(t, []scriptedQuery{
468+
{
469+
match: expectContains(`SELECT queue_name FROM absurd.queues ORDER BY queue_name`),
470+
columns: []string{"queue_name"},
471+
rows: [][]driver.Value{{"alpha"}},
472+
},
473+
{
474+
match: expectRecentTaskNamesQuery("alpha", 5000),
475+
columns: []string{"task_name"},
476+
rows: [][]driver.Value{{"process-webhook"}},
477+
},
478+
{
479+
match: expectFailedQueueTasksQuery("alpha", 27, false),
480+
columns: []string{
481+
"task_id",
482+
"run_id",
483+
"queue_name",
484+
"task_name",
485+
"state",
486+
"attempt",
487+
"max_attempts",
488+
"created_at",
489+
"updated_at",
490+
"completed_at",
491+
"claimed_by",
492+
"params",
493+
},
494+
rows: [][]driver.Value{
495+
{
496+
taskID,
497+
runID,
498+
"alpha",
499+
"process-webhook",
500+
"failed",
501+
int64(1),
502+
int64(3),
503+
now,
504+
now,
505+
nil,
506+
nil,
507+
nil,
508+
},
509+
},
510+
},
511+
})
512+
513+
srv := &Server{db: db}
514+
515+
req := httptest.NewRequest(http.MethodGet, "/api/tasks?status=failed&failedRunScope=all&page=1&perPage=25", nil)
516+
resp := httptest.NewRecorder()
517+
518+
srv.handleTasks(resp, req)
519+
520+
if resp.Code != http.StatusOK {
521+
t.Fatalf("status = %d, want %d (body=%q)", resp.Code, http.StatusOK, resp.Body.String())
522+
}
523+
524+
body := resp.Body.String()
525+
if !strings.Contains(body, runID) {
526+
t.Fatalf("expected failed run in response body, got %q", body)
527+
}
528+
}
529+
530+
func TestHandleTasksRejectsInvalidFailedRunScope(t *testing.T) {
531+
db := newScriptedDB(t, []scriptedQuery{
532+
{
533+
match: expectContains(`SELECT queue_name FROM absurd.queues ORDER BY queue_name`),
534+
columns: []string{"queue_name"},
535+
rows: [][]driver.Value{{"alpha"}},
536+
},
537+
})
538+
539+
srv := &Server{db: db}
540+
541+
req := httptest.NewRequest(http.MethodGet, "/api/tasks?failedRunScope=sometimes&page=1&perPage=25", nil)
542+
resp := httptest.NewRecorder()
543+
544+
srv.handleTasks(resp, req)
545+
546+
if resp.Code != http.StatusOK {
547+
t.Fatalf("status = %d, want %d (body=%q)", resp.Code, http.StatusOK, resp.Body.String())
548+
}
549+
550+
body := resp.Body.String()
551+
if !strings.Contains(body, `"items":[]`) || !strings.Contains(body, `"total":0`) {
552+
t.Fatalf("expected empty task list response, got %q", body)
553+
}
554+
}
555+
355556
func TestHandleRetryTaskSuccess(t *testing.T) {
356557
taskID := uuid.NewString()
357558
runID := uuid.NewString()

‎habitat/ui/src/lib/api.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -159,9 +159,12 @@ export interface TaskListResponse {
159159
availableTaskNames: string[];
160160
}
161161

162+
export type FailedRunScope = "all" | "terminal";
163+
162164
export interface TaskListQuery {
163165
search?: string;
164166
status?: string | null;
167+
failedRunScope?: FailedRunScope | null;
165168
queue?: string | null;
166169
taskName?: string | null;
167170
taskId?: string | null;
@@ -182,6 +185,9 @@ export async function fetchTasks(
182185
if (filters.status) {
183186
params.set("status", filters.status);
184187
}
188+
if (filters.failedRunScope) {
189+
params.set("failedRunScope", filters.failedRunScope);
190+
}
185191
if (filters.queue) {
186192
params.set("queue", filters.queue);
187193
}

0 commit comments

Comments
 (0)