fix(telemetry): send browser events to the configured collector and keep logs off the anonymous one (#9317)

* fix(telemetry): send browser events to the configured collector and keep logs off the anonymous one

The browser telemetry route read only TELEMETRY_ENDPOINT, so a deployment exporting to its own
collector through OTEL_EXPORTER_OTLP_ENDPOINT still posted browser events to the anonymous
default, ignored NEXT_TELEMETRY_DISABLED, and built spans without trace or span ids under the
retired instrumentationLibrarySpans field. It now resolves the endpoint and headers exactly like
the server export, honors the disable flag, and emits valid OTLP. Server logs and metrics are no
longer exported to the anonymous collector, which receives traces only.

* fix(telemetry): never send collector credentials to the anonymous endpoint, match it by host

OTEL_EXPORTER_OTLP_HEADERS belong to the operator's collector, so they are dropped when the
endpoint resolves to the anonymous one; that check now compares hosts, so any spelling of the
anonymous URL gets traces only. The browser relay falls back to telemetry.config.ts like the
server export, and the traces URL is built on the parsed path, keeping query strings and
accepting a trailing slash.

* fix(telemetry): never attach operator collector credentials to browser events

/api/telemetry relays unauthenticated input, so sending it with OTEL_EXPORTER_OTLP_HEADERS let
anyone write spans into the operator's authenticated collector. Browser events now go only to
TELEMETRY_ENDPOINT or the configured anonymous collector, without credentials; the server's own
export keeps using the operator's collector.

* fix(telemetry): post browser events to TELEMETRY_ENDPOINT as-is and document routing per side

Author: waleedlatif1

apps/docs/content/docs/platform/self-hosting/environment-variables.mdx

@@ -298,7 +298,7 @@ helm template sim oci://ghcr.io/simstudioai/charts/sim --version 1.11.8 -f value
 | `OTEL_TRACES_SAMPLER_ARG` | Trace sampling ratio |
 | `OTEL_DEPLOYMENT_ENVIRONMENT` | Environment label on emitted spans |
 | `TELEMETRY_SAMPLING_RATIO` | Application-level sampling ratio |
-| `TELEMETRY_ENDPOINT` | Where anonymous telemetry is sent. Defaults to `https://telemetry.simstudio.ai/v1/traces` — point it at your own collector to keep traces internal |
+| `TELEMETRY_ENDPOINT` | Full URL browser telemetry events are posted to, as-is (they never carry `OTEL_EXPORTER_OTLP_HEADERS`). Server traces use it only when `OTEL_EXPORTER_OTLP_ENDPOINT` is unset. Defaults to `https://telemetry.simstudio.ai/v1/traces`, which receives traces only: server logs and metrics are exported only to a collector you configure. `NEXT_TELEMETRY_DISABLED=1` (the Docker and Helm default) turns all of it off |
 | `NEXT_TELEMETRY_DISABLED` | Set to `1` to disable anonymous telemetry entirely |
 | `GRAFANA_OTLP_ENDPOINT` / `GRAFANA_OTLP_HEADERS` / `GRAFANA_DEPLOYMENT_ENVIRONMENT` | Grafana Cloud OTLP export |
 

apps/sim/app/api/telemetry/route.ts

@@ -1,12 +1,14 @@
 import { createLogger } from '@sim/logger'
 import { getErrorMessage } from '@sim/utils/errors'
+import { generateRandomHex } from '@sim/utils/random'
 import { type NextRequest, NextResponse } from 'next/server'
 import { telemetryContract } from '@/lib/api/contracts/telemetry'
 import { parseRequest } from '@/lib/api/server'
-import { env } from '@/lib/core/config/env'
 import { isProd } from '@/lib/core/config/env-flags'
 import { enforceIpRateLimit } from '@/lib/core/rate-limiter'
 import { withRouteHandler } from '@/lib/core/utils/with-route-handler'
+import { isTelemetryDisabled, normalizeOtlpTracesUrl } from '@/lib/monitoring/otlp'
+import telemetryConfig from '@/telemetry.config'
 
 const logger = createLogger('TelemetryAPI')
 
@@ -52,15 +54,20 @@ function createSafeAttributes(
 }
 
 /**
- * Forwards telemetry data to OpenTelemetry collector
+ * Forwards a browser telemetry event as one span, and not at all when the operator turned telemetry
+ * off. Browser events are unauthenticated input, so they never travel with the operator's
+ * collector credentials: they go to `TELEMETRY_ENDPOINT` or the configured anonymous collector.
  */
 async function forwardToCollector(data: Record<string, unknown>): Promise<boolean> {
   if (!data || typeof data !== 'object') {
     logger.error('Invalid telemetry data format')
     return false
   }
+  if (isTelemetryDisabled()) return false
 
-  const endpoint = env.TELEMETRY_ENDPOINT || 'https://telemetry.simstudio.ai/v1/traces'
+  // TELEMETRY_ENDPOINT has always been the relay's full destination URL, so it is posted to as-is.
+  const endpoint =
+    process.env.TELEMETRY_ENDPOINT || normalizeOtlpTracesUrl(telemetryConfig.endpoint)
   const timeout = DEFAULT_TIMEOUT
 
   try {
@@ -89,10 +96,12 @@ async function forwardToCollector(data: Record<string, unknown>): Promise<boolea
           resource: {
             attributes: serviceAttrs,
           },
-          instrumentationLibrarySpans: [
+          scopeSpans: [
             {
               spans: [
                 {
+                  traceId: generateRandomHex(32),
+                  spanId: generateRandomHex(16),
                   name: spanName,
                   kind: 1,
                   startTimeUnixNano: timestamp,

apps/sim/instrumentation-node.ts

@@ -15,7 +15,12 @@ import type {
 import { createLogger } from '@sim/logger'
 import { TraceAttr } from '@/lib/mothership/generated/trace-attributes-v1'
 import { env } from './lib/core/config/env'
-import { parseOtlpHeaders } from './lib/monitoring/otlp'
+import {
+  ANONYMOUS_TELEMETRY_ENDPOINT,
+  isTelemetryDisabled,
+  normalizeOtlpTracesUrl,
+  resolveOtlpTarget,
+} from './lib/monitoring/otlp'
 
 diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.ERROR)
 
@@ -27,7 +32,7 @@ const SPAN_NAME_PREFIX = `${MOTHERSHIP_ORIGIN}: `
 const SERVICE_INSTANCE_SLUG = 'sim' as const
 
 const DEFAULT_TELEMETRY_CONFIG = {
-  endpoint: env.TELEMETRY_ENDPOINT || 'https://telemetry.simstudio.ai/v1/traces',
+  endpoint: ANONYMOUS_TELEMETRY_ENDPOINT,
   serviceName: 'mothership',
   serviceVersion: '0.1.0',
   serverSide: { enabled: true },
@@ -49,21 +54,6 @@ function isBusinessSpan(spanName: string): boolean {
   return ALLOWED_SPAN_PREFIXES.some((prefix) => spanName.startsWith(prefix))
 }
 
-// Append `/v1/traces` to the OTLP base URL unless already present.
-// The HTTP exporter doesn't auto-suffix the signal path even though
-// the spec says the env var is a base URL.
-function normalizeOtlpTracesUrl(url: string): string {
-  if (!url) return url
-  try {
-    const u = new URL(url)
-    if (u.pathname.endsWith('/v1/traces')) return url
-    const base = url.replace(/\/$/, '')
-    return `${base}/v1/traces`
-  } catch {
-    return url
-  }
-}
-
 // Metrics counterpart to `normalizeOtlpTracesUrl`. Operates on the parsed
 // pathname (not a raw string suffix) so query strings and trailing slashes
 // don't corrupt the result: swap a `/v1/traces` suffix for `/v1/metrics`,
@@ -151,7 +141,7 @@ class MothershipOriginSpanProcessor implements SpanProcessor {
 
 async function initializeOpenTelemetry() {
   try {
-    if (env.NEXT_TELEMETRY_DISABLED === '1' || process.env.NEXT_TELEMETRY_DISABLED === '1') {
+    if (isTelemetryDisabled()) {
       logger.info('OpenTelemetry disabled via NEXT_TELEMETRY_DISABLED=1')
       return
     }
@@ -163,15 +153,10 @@ async function initializeOpenTelemetry() {
       telemetryConfig = DEFAULT_TELEMETRY_CONFIG
     }
 
-    // Prefer the OTel spec env var, fall back to legacy TELEMETRY_ENDPOINT.
-    const resolvedEndpoint =
-      process.env.OTEL_EXPORTER_OTLP_ENDPOINT ||
-      process.env.TELEMETRY_ENDPOINT ||
-      env.TELEMETRY_ENDPOINT ||
-      telemetryConfig.endpoint
+    const target = resolveOtlpTarget(telemetryConfig.endpoint)
     telemetryConfig = {
       ...telemetryConfig,
-      endpoint: resolvedEndpoint,
+      endpoint: target.endpoint,
       serviceName: 'mothership',
     }
 
@@ -244,7 +229,7 @@ async function initializeOpenTelemetry() {
       },
     })
 
-    const otlpHeaders = parseOtlpHeaders(process.env.OTEL_EXPORTER_OTLP_HEADERS || '')
+    const otlpHeaders = target.headers
     const exporterUrl = normalizeOtlpTracesUrl(telemetryConfig.endpoint)
 
     const exporter = new OTLPTraceExporter({
@@ -279,28 +264,33 @@ async function initializeOpenTelemetry() {
 
     // Metrics (hosted-key counters/histograms) share the trace endpoint and
     // headers — only the signal path differs. Unlike spans these aren't sampled.
-    const metricReader = new PeriodicExportingMetricReader({
-      exporter: new OTLPMetricExporter({
-        url: normalizeOtlpMetricsUrl(telemetryConfig.endpoint),
-        headers: otlpHeaders,
-        timeoutMillis: Math.min(telemetryConfig.batchSettings.exportTimeoutMillis, 10000),
-        keepAlive: false,
-      }),
-      exportIntervalMillis: 60000,
-    })
+    // Neither metrics nor logs go to the anonymous collector, which takes traces only.
+    const metricReader = target.anonymous
+      ? undefined
+      : new PeriodicExportingMetricReader({
+          exporter: new OTLPMetricExporter({
+            url: normalizeOtlpMetricsUrl(telemetryConfig.endpoint),
+            headers: otlpHeaders,
+            timeoutMillis: Math.min(telemetryConfig.batchSettings.exportTimeoutMillis, 10000),
+            keepAlive: false,
+          }),
+          exportIntervalMillis: 60000,
+        })
 
     // Logs share the trace endpoint and headers as well (signal path
     // /v1/logs). Every @sim/logger line fans out through the global Logs API
     // (see packages/logger), which the NodeSDK wires to this processor — the
     // stdout JSON lines continue to CloudWatch unchanged.
-    const logRecordProcessor = new BatchLogRecordProcessor({
-      exporter: new OTLPLogExporter({
-        url: normalizeOtlpLogsUrl(telemetryConfig.endpoint),
-        headers: otlpHeaders,
-        timeoutMillis: Math.min(telemetryConfig.batchSettings.exportTimeoutMillis, 10000),
-        keepAlive: false,
-      }),
-    })
+    const logRecordProcessor = target.anonymous
+      ? undefined
+      : new BatchLogRecordProcessor({
+          exporter: new OTLPLogExporter({
+            url: normalizeOtlpLogsUrl(telemetryConfig.endpoint),
+            headers: otlpHeaders,
+            timeoutMillis: Math.min(telemetryConfig.batchSettings.exportTimeoutMillis, 10000),
+            keepAlive: false,
+          }),
+        })
 
     // Must be unique per process: replicas sharing one instance id collapse
     // into a single Prometheus series, so their independent cumulative
@@ -350,8 +340,9 @@ async function initializeOpenTelemetry() {
       resource,
       spanProcessors,
       sampler,
-      metricReader,
-      logRecordProcessors: [logRecordProcessor],
+      // Explicit empty lists: left undefined, NodeSDK builds default OTLP exporters from env.
+      metricReaders: metricReader ? [metricReader] : [],
+      logRecordProcessors: logRecordProcessor ? [logRecordProcessor] : [],
     })
 
     sdk.start()

apps/sim/lib/monitoring/otlp.ts

@@ -25,3 +25,62 @@ export function parseOtlpHeaders(raw: string): Record<string, string> {
   }
   return out
 }
+
+/**
+ * Sim's anonymous telemetry collector, used when no collector is configured. It receives traces
+ * only: logs and metrics carry an instance's own operational detail and go only to a collector
+ * its operator chose.
+ */
+export const ANONYMOUS_TELEMETRY_ENDPOINT = 'https://telemetry.simstudio.ai/v1/traces'
+
+/** True when the operator turned telemetry off (`NEXT_TELEMETRY_DISABLED=1`). */
+export function isTelemetryDisabled(): boolean {
+  return process.env.NEXT_TELEMETRY_DISABLED === '1'
+}
+
+/** True when `endpoint` names the anonymous collector's host, however the URL is spelled. */
+function isAnonymousCollector(endpoint: string): boolean {
+  try {
+    return new URL(endpoint).host === new URL(ANONYMOUS_TELEMETRY_ENDPOINT).host
+  } catch {
+    return false
+  }
+}
+
+/**
+ * The collector this process sends telemetry to: `OTEL_EXPORTER_OTLP_ENDPOINT`, then the legacy
+ * `TELEMETRY_ENDPOINT`, then `fallback` (a fork's `telemetry.config.ts` endpoint, or the anonymous
+ * collector). `OTEL_EXPORTER_OTLP_HEADERS` belong to the operator's own collector, so they are
+ * never sent to the anonymous one.
+ */
+export function resolveOtlpTarget(fallback: string = ANONYMOUS_TELEMETRY_ENDPOINT): {
+  endpoint: string
+  headers: Record<string, string>
+  anonymous: boolean
+} {
+  const endpoint =
+    process.env.OTEL_EXPORTER_OTLP_ENDPOINT || process.env.TELEMETRY_ENDPOINT || fallback
+  const anonymous = isAnonymousCollector(endpoint)
+  return {
+    endpoint,
+    headers: anonymous ? {} : parseOtlpHeaders(process.env.OTEL_EXPORTER_OTLP_HEADERS || ''),
+    anonymous,
+  }
+}
+
+/**
+ * Appends `/v1/traces` to an OTLP base URL's path unless it already ends there (with or without a
+ * trailing slash), keeping any query string. The HTTP exporter doesn't auto-suffix the signal path
+ * even though the spec says the env var is a base URL.
+ */
+export function normalizeOtlpTracesUrl(url: string): string {
+  if (!url) return url
+  try {
+    const u = new URL(url)
+    const path = u.pathname.replace(/\/+$/, '')
+    u.pathname = path.endsWith('/v1/traces') ? path : `${path}/v1/traces`
+    return u.toString()
+  } catch {
+    return url
+  }
+}

apps/sim/telemetry.config.ts

@@ -23,15 +23,15 @@
  * - API keys or tokens
  * - IP addresses or geolocation data
  */
-import { env } from './lib/core/config/env'
+import { ANONYMOUS_TELEMETRY_ENDPOINT } from './lib/monitoring/otlp'
 
 const config = {
   /**
-   * OTLP Endpoint URL where telemetry data is sent
-   * Change this if you want to send telemetry to your own collector
+   * OTLP Endpoint URL where telemetry data is sent when neither OTEL_EXPORTER_OTLP_ENDPOINT nor
+   * TELEMETRY_ENDPOINT is set. Change this if you want to send telemetry to your own collector.
    * Supports any OTLP-compatible backend (Jaeger, Grafana Tempo, etc.)
    */
-  endpoint: env.TELEMETRY_ENDPOINT || 'https://telemetry.simstudio.ai/v1/traces',
+  endpoint: ANONYMOUS_TELEMETRY_ENDPOINT,
 
   /**
    * Service name used to identify this instance