perf(webapp): bound finished-run trace reads by completion time

Reads of a finished run's trace events on the run page, in span details, the run events API and the dashboard agent's run trace are now bounded to events stored up to 7 days after the run completed, so opening an old run no longer scans every day of stored events since it started. Running runs stay unbounded and the logs download is unchanged.

Mono-RevId: daaf4338a262842b760a4015cfe208649787451a

Author: Mariefay

.server-changes/older-run-traces-faster.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Older runs now load their trace, span details and run events faster. A span that finishes more than a week after its run shows as still in progress.

apps/webapp/app/presenters/v3/RunPresenter.server.ts

@@ -12,7 +12,7 @@ import { buildTraceView } from "~/v3/eventRepository/traceViewBuilder";
 import { runStore } from "~/v3/runStore.server";
 import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server";
 import { runTriggeredAt } from "~/v3/runTimestamps";
-import { getRunFinishedAt } from "~/v3/eventRepository/traceInsertedAtBound";
+import { getRunFinishedAt, getTraceInsertedAtEnd } from "~/v3/eventRepository/traceInsertedAtBound";
 
 export class RunEnvironmentMismatchError extends Error {
   constructor(message: string) {
@@ -193,14 +193,15 @@ export class RunPresenter {
     );
 
     const traceScope = runTraceReadScope(run, environment.id);
+    const insertedAtEnd = getTraceInsertedAtEnd(run);
     const queuedDuration = runQueuedDuration(run);
 
     const firstChunkPromise = canLiveTail(
       run.taskEventStore,
       env.TRACE_VIEW_EMERGENCY_SPAN_CAP,
       env.EVENT_REPOSITORY_V1_REROUTE_ENABLED
     )
-      ? readFirstTraceChunk({ repository, scope: traceScope, showDebug })
+      ? readFirstTraceChunk({ repository, scope: traceScope, showDebug, insertedAtEnd })
       : Promise.resolve(undefined);
 
     const [user, firstRead] = await Promise.all([
@@ -220,6 +221,7 @@ export class RunPresenter {
           firstRead,
           anchorSpanId: run.spanId,
           selectedSpanId,
+          insertedAtEnd,
           buildOptions,
           startedAt: run.startedAt,
           queuedDuration,
@@ -259,7 +261,7 @@ export class RunPresenter {
           run.traceId,
           traceScope.startCreatedAt,
           traceScope.endCreatedAt,
-          { includeDebugLogs: showDebug, anchorSpanId: run.spanId }
+          { includeDebugLogs: showDebug, anchorSpanId: run.spanId, insertedAtEnd }
         );
 
     let isTruncated = traceSummary?.isTruncated ?? false;
@@ -273,7 +275,7 @@ export class RunPresenter {
         traceScope.startCreatedAt,
         traceScope.endCreatedAt,
         // The anchor is the viewed run: a root has no ancestors, a child doesn't need them.
-        { includeDebugLogs: showDebug }
+        { includeDebugLogs: showDebug, insertedAtEnd }
       );
 
       if (subtreeSummary) {

apps/webapp/app/presenters/v3/SpanPresenter.server.ts

@@ -52,7 +52,10 @@ import {
   type ResolvedRunLockedWorker,
 } from "~/v3/runOpsMigration/controlPlaneResolver.server";
 import type { AuthenticatedEnvironment } from "@trigger.dev/core/v3/auth/environment";
-import { getTraceStartTimeEnd } from "~/v3/eventRepository/traceInsertedAtBound";
+import {
+  getTraceInsertedAtEnd,
+  getTraceStartTimeEnd,
+} from "~/v3/eventRepository/traceInsertedAtBound";
 import {
   findStartedSession,
   startedSessionExternalId,
@@ -238,6 +241,7 @@ export class SpanPresenter extends BasePresenter {
       projectId: parentRun.projectId,
       createdAt: parentRun.createdAt,
       endCreatedAt: getTraceStartTimeEnd(parentRun),
+      insertedAtEnd: getTraceInsertedAtEnd(parentRun),
       eventRepository: repository,
       showDebug,
       rootRunId: traceRootRunId(parentRun),
@@ -706,6 +710,7 @@ export class SpanPresenter extends BasePresenter {
     projectId,
     createdAt,
     endCreatedAt,
+    insertedAtEnd,
     showDebug,
     rootRunId,
     userId,
@@ -718,6 +723,7 @@ export class SpanPresenter extends BasePresenter {
     eventStore: TaskEventStoreTable;
     createdAt: Date;
     endCreatedAt: Date | undefined;
+    insertedAtEnd: Date | undefined;
     showDebug: boolean;
     rootRunId: string;
     userId: string;
@@ -729,7 +735,7 @@ export class SpanPresenter extends BasePresenter {
       traceId,
       createdAt,
       endCreatedAt,
-      { includeDebugLogs: showDebug }
+      { includeDebugLogs: showDebug, insertedAtEnd }
     );
 
     if (!span) {

apps/webapp/app/presenters/v3/TraceChunkPresenter.server.ts

@@ -62,8 +62,6 @@ const startRunSelect = {
   annotations: true,
 } satisfies Prisma.TaskRunSelect;
 
-type ChunkRun = Prisma.TaskRunGetPayload<{ select: typeof chunkRunSelect }>;
-
 type TraceChunkPresenterDeps = {
   prismaClient: PrismaClient;
   runStore: Pick<RunStore, "findRun">;
@@ -137,7 +135,7 @@ export class TraceChunkPresenter {
     );
 
     const scope = runTraceReadScope(run, environment.id);
-    const insertedAtEnd = traceInsertedAtEndFor(run, context);
+    const insertedAtEnd = getTraceInsertedAtEnd(run);
     recordReadAttributes(context, insertedAtEnd);
 
     if (filter === "errors") {
@@ -232,7 +230,7 @@ export class TraceChunkPresenter {
     );
 
     const scope = runTraceReadScope(run, environment.id);
-    const insertedAtEnd = traceInsertedAtEndFor(run, context);
+    const insertedAtEnd = getTraceInsertedAtEnd(run);
     recordReadAttributes(context, insertedAtEnd);
 
     const firstRead = await readFirstTraceChunk({ repository, scope, showDebug, insertedAtEnd });
@@ -295,11 +293,6 @@ export class TraceChunkPresenter {
   }
 }
 
-// The run page reads unbounded: a child finishing long after the run would lose its final row.
-function traceInsertedAtEndFor(run: ChunkRun, context: TraceReadContext | undefined) {
-  return context === "session" ? getTraceInsertedAtEnd(run) : undefined;
-}
-
 function recordReadAttributes(
   context: TraceReadContext | undefined,
   insertedAtEnd: Date | undefined

apps/webapp/app/routes/api.v1.runs.$runId.events.ts

@@ -4,7 +4,10 @@ import { getTaskEventStoreTableForRun } from "~/v3/taskEventStore.server";
 import { anyResource, createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server";
 import { ApiRetrieveRunPresenter } from "~/presenters/v3/ApiRetrieveRunPresenter.server";
 import { getEventRepositoryForStore } from "~/v3/eventRepository/index.server";
-import { getTraceStartTimeEnd } from "~/v3/eventRepository/traceInsertedAtBound";
+import {
+  getTraceInsertedAtEnd,
+  getTraceStartTimeEnd,
+} from "~/v3/eventRepository/traceInsertedAtBound";
 
 const ParamsSchema = z.object({
   runId: z.string(), // This is the run friendly ID
@@ -57,7 +60,8 @@ export const loader = createLoaderApiRoute(
       run.traceId,
       run.friendlyId,
       run.createdAt,
-      getTraceStartTimeEnd(run)
+      getTraceStartTimeEnd(run),
+      { insertedAtEnd: getTraceInsertedAtEnd(run) }
     );
 
     // TODO: return only relevant fields, avoid returning the whole events

apps/webapp/app/routes/api.v1.runs.$runId.spans.$spanId.ts

@@ -5,7 +5,10 @@ import { $replica } from "~/db.server";
 import { extractAISpanData } from "~/components/runs/v3/ai";
 import { anyResource, createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server";
 import { getEventRepositoryForStore } from "~/v3/eventRepository/index.server";
-import { getTraceStartTimeEnd } from "~/v3/eventRepository/traceInsertedAtBound";
+import {
+  getTraceInsertedAtEnd,
+  getTraceStartTimeEnd,
+} from "~/v3/eventRepository/traceInsertedAtBound";
 import { getTaskEventStoreTableForRun } from "~/v3/taskEventStore.server";
 import { findRunByIdWithMollifierFallback } from "~/v3/mollifier/readFallback.server";
 import { buildSyntheticSpanDetailBody } from "~/v3/mollifier/syntheticApiResponses.server";
@@ -107,7 +110,8 @@ export const loader = createLoaderApiRoute(
       params.spanId,
       run.traceId,
       run.createdAt,
-      getTraceStartTimeEnd(run)
+      getTraceStartTimeEnd(run),
+      { insertedAtEnd: getTraceInsertedAtEnd(run) }
     );
 
     if (!span) {

apps/webapp/app/services/dashboardAgentRunTrace.server.ts

@@ -7,7 +7,10 @@ import { logger } from "~/services/logger.server";
 import { getAdminOnlyForEvent } from "~/utils/timelineSpanEvents";
 import { type SpanSummary, type TraceSummary } from "~/v3/eventRepository/eventRepository.types";
 import { getEventRepositoryForStore } from "~/v3/eventRepository/index.server";
-import { getTraceStartTimeEnd } from "~/v3/eventRepository/traceInsertedAtBound";
+import {
+  getTraceInsertedAtEnd,
+  getTraceStartTimeEnd,
+} from "~/v3/eventRepository/traceInsertedAtBound";
 import { runStore } from "~/v3/runStore.server";
 import { getTaskEventStoreTableForRun } from "~/v3/taskEventStore.server";
 
@@ -66,14 +69,15 @@ export async function getDashboardAgentRunTrace({
   const repository = await getEventRepositoryForStore(run.taskEventStore, organizationId);
   const storeTable = getTaskEventStoreTableForRun(run);
   const endCreatedAt = getTraceStartTimeEnd(run);
+  const insertedAtEnd = getTraceInsertedAtEnd(run);
 
   let summary: TraceSummary | undefined = await repository.getTraceSummary(
     storeTable,
     environmentId,
     run.traceId,
     run.createdAt,
     endCreatedAt,
-    { includeDebugLogs: false, anchorSpanId: run.spanId }
+    { includeDebugLogs: false, anchorSpanId: run.spanId, insertedAtEnd }
   );
 
   // No summary means the anchor span isn't in the summary rows, e.g. it fell past the
@@ -87,7 +91,7 @@ export async function getDashboardAgentRunTrace({
       run.createdAt,
       endCreatedAt,
       // The tree is built from the anchor down, so ancestors would be read for nothing.
-      { includeDebugLogs: false }
+      { includeDebugLogs: false, insertedAtEnd }
     );
 
     if (subtree) {

apps/webapp/app/v3/eventRepository/clickhouseEventRepository.server.ts

@@ -1696,7 +1696,7 @@ export class ClickhouseEventRepository implements IEventRepository {
     traceId: string,
     startCreatedAt: Date,
     endCreatedAt?: Date,
-    options?: { includeDebugLogs?: boolean; anchorSpanId?: string }
+    options?: { includeDebugLogs?: boolean; anchorSpanId?: string; insertedAtEnd?: Date }
   ): Promise<TraceSummary | undefined> {
     const limit = this._config.maximumTraceSummaryViewCount;
     const records = await this.#fetchTraceSummaryRecords({
@@ -1741,7 +1741,7 @@ export class ClickhouseEventRepository implements IEventRepository {
     anchorSpanId: string,
     startCreatedAt: Date,
     endCreatedAt?: Date,
-    options?: { includeDebugLogs?: boolean }
+    options?: { includeDebugLogs?: boolean; insertedAtEnd?: Date }
   ): Promise<TraceSummary | undefined> {
     const { records, isTruncated, missingAnchor } = await this.#fetchTraceSubtreeRecords({
       environmentId,
@@ -1927,6 +1927,17 @@ export class ClickhouseEventRepository implements IEventRepository {
     queryBuilder.where(sortKey.clause, sortKey.params);
   }
 
+  #whereInsertedAtEnd(
+    queryBuilder: { where: (clause: string, params?: any) => unknown },
+    end: Date | undefined
+  ) {
+    if (end) {
+      queryBuilder.where("inserted_at <= {insertedAtEnd: DateTime64(3)}", {
+        insertedAtEnd: convertDateToClickhouseDateTime(end),
+      });
+    }
+  }
+
   #applyTraceScopeWhere(
     queryBuilder: { where: (clause: string, params?: any) => unknown },
     {
@@ -1968,11 +1979,7 @@ export class ClickhouseEventRepository implements IEventRepository {
         });
       }
 
-      if (options?.insertedAtEnd) {
-        queryBuilder.where("inserted_at <= {insertedAtEnd: DateTime64(3)}", {
-          insertedAtEnd: convertDateToClickhouseDateTime(options.insertedAtEnd),
-        });
-      }
+      this.#whereInsertedAtEnd(queryBuilder, options?.insertedAtEnd);
     }
 
     if (options?.includeDebugLogs !== true) {
@@ -2172,7 +2179,7 @@ export class ClickhouseEventRepository implements IEventRepository {
     anchorSpanId: string;
     startCreatedAt: Date;
     endCreatedAt?: Date;
-    options?: { includeDebugLogs?: boolean };
+    options?: { includeDebugLogs?: boolean; insertedAtEnd?: Date };
     limit?: number;
   }): Promise<{
     records: TaskEventSummaryV1Result[];
@@ -2419,7 +2426,7 @@ export class ClickhouseEventRepository implements IEventRepository {
     traceId: string;
     startCreatedAt?: Date;
     endCreatedAt?: Date;
-    options?: { includeDebugLogs?: boolean };
+    options?: { includeDebugLogs?: boolean; insertedAtEnd?: Date };
     spanIds?: string[];
     parentSpanIds?: string[];
     limit?: number;
@@ -2446,6 +2453,8 @@ export class ClickhouseEventRepository implements IEventRepository {
         queryBuilder.where("inserted_at >= {insertedAtStart: DateTime64(3)}", {
           insertedAtStart: convertDateToClickhouseDateTime(startCreatedAtWithBuffer),
         });
+
+        this.#whereInsertedAtEnd(queryBuilder, options?.insertedAtEnd);
       }
     }
 
@@ -2542,7 +2551,7 @@ export class ClickhouseEventRepository implements IEventRepository {
     traceId: string,
     startCreatedAt: Date,
     endCreatedAt?: Date,
-    options?: { includeDebugLogs?: boolean }
+    options?: { includeDebugLogs?: boolean; insertedAtEnd?: Date }
   ): Promise<SpanDetail | undefined> {
     const startCreatedAtWithBuffer = new Date(startCreatedAt.getTime() - 60_000);
 
@@ -2561,6 +2570,8 @@ export class ClickhouseEventRepository implements IEventRepository {
       queryBuilder.where("inserted_at >= {insertedAtStart: DateTime64(3)}", {
         insertedAtStart: convertDateToClickhouseDateTime(startCreatedAtWithBuffer),
       });
+
+      this.#whereInsertedAtEnd(queryBuilder, options?.insertedAtEnd);
     }
 
     if (options?.includeDebugLogs !== true) {
@@ -3058,11 +3069,7 @@ export class ClickhouseEventRepository implements IEventRepository {
           insertedAtStart: convertDateToClickhouseDateTime(startCreatedAtWithBuffer),
         });
 
-        if (options?.insertedAtEnd) {
-          queryBuilder.where("inserted_at <= {insertedAtEnd: DateTime64(3)}", {
-            insertedAtEnd: convertDateToClickhouseDateTime(options.insertedAtEnd),
-          });
-        }
+        this.#whereInsertedAtEnd(queryBuilder, options?.insertedAtEnd);
       }
     } else if (insertedAtStart && this._version === "v2") {
       queryBuilder.where("inserted_at >= {insertedAtStart: DateTime64(3)}", {
@@ -3484,7 +3491,8 @@ export class ClickhouseEventRepository implements IEventRepository {
     traceId: string,
     runId: string,
     startCreatedAt: Date,
-    endCreatedAt?: Date
+    endCreatedAt?: Date,
+    options?: { insertedAtEnd?: Date }
   ): Promise<RunPreparedEvent[]> {
     const startCreatedAtWithBuffer = new Date(startCreatedAt.getTime() - 1000);
 
@@ -3503,6 +3511,8 @@ export class ClickhouseEventRepository implements IEventRepository {
       queryBuilder.where("inserted_at >= {insertedAtStart: DateTime64(3)}", {
         insertedAtStart: convertDateToClickhouseDateTime(startCreatedAtWithBuffer),
       });
+
+      this.#whereInsertedAtEnd(queryBuilder, options?.insertedAtEnd);
     }
 
     queryBuilder.where("kind != {kind: String}", { kind: "DEBUG_EVENT" });

apps/webapp/app/v3/eventRepository/eventRepository.server.ts

@@ -453,7 +453,7 @@ export class EventRepository implements IEventRepository {
     traceId: string,
     startCreatedAt: Date,
     endCreatedAt?: Date,
-    options?: { includeDebugLogs?: boolean; anchorSpanId?: string }
+    options?: { includeDebugLogs?: boolean; anchorSpanId?: string; insertedAtEnd?: Date }
   ): Promise<TraceSummary | undefined> {
     return await startActiveSpan("getTraceSummary", async (span) => {
       const events = await this.taskEventStore.findTraceEvents(
@@ -485,7 +485,7 @@ export class EventRepository implements IEventRepository {
     _anchorSpanId: string,
     _startCreatedAt: Date,
     _endCreatedAt?: Date,
-    _options?: { includeDebugLogs?: boolean }
+    _options?: { includeDebugLogs?: boolean; insertedAtEnd?: Date }
   ): Promise<TraceSummary | undefined> {
     // Subtree traversal is ClickHouse-only. Dashboard falls back to the full
     // summary when this returns undefined.
@@ -625,7 +625,8 @@ export class EventRepository implements IEventRepository {
     traceId: string,
     runId: string,
     startCreatedAt: Date,
-    endCreatedAt?: Date
+    endCreatedAt?: Date,
+    _options?: { insertedAtEnd?: Date }
   ): Promise<RunPreparedEvent[]> {
     return await startActiveSpan("getRunEvents", async (span) => {
       const events = await this.taskEventStore.findMany(
@@ -680,7 +681,7 @@ export class EventRepository implements IEventRepository {
     traceId: string,
     startCreatedAt: Date,
     endCreatedAt?: Date,
-    options?: { includeDebugLogs?: boolean }
+    options?: { includeDebugLogs?: boolean; insertedAtEnd?: Date }
   ): Promise<SpanDetail | undefined> {
     return await startActiveSpan("getSpan", async (s) => {
       const spanEvent = await this.#getSpanEvent({