fix: various bugs in observability / OTel (#1564)

frontend/package.json

@@ -14,6 +14,11 @@
     "format": "prettier --write ."
   },
   "dependencies": {
+    "@opentelemetry/api": "^1.9.1",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.219.0",
+    "@opentelemetry/resources": "^2.8.0",
+    "@opentelemetry/sdk-trace-web": "^2.8.0",
+    "@opentelemetry/semantic-conventions": "^1.41.1",
     "@simplewebauthn/browser": "^13.3.0",
     "@tailwindcss/vite": "^4.3.0",
     "axios": "^1.16.1",

frontend/src/lib/services/api-service.ts

@@ -1,4 +1,9 @@
-import axios from 'axios';
+import { endRequestSpan, startRequestSpan } from '$lib/utils/tracing-util';
+import axios, { AxiosError, type InternalAxiosRequestConfig } from 'axios';
+import type { Span } from '@opentelemetry/api';
+
+// Tracks the in-flight span for each request config so the response/error interceptor can end it.
+const requestSpans = new WeakMap<InternalAxiosRequestConfig, Span>();
 
 abstract class APIService {
 	protected api = axios.create({ baseURL: '/api' });
@@ -7,6 +12,32 @@ abstract class APIService {
 		if (typeof process !== 'undefined' && process?.env?.DEVELOPMENT_BACKEND_URL) {
 			this.api.defaults.baseURL = process.env.DEVELOPMENT_BACKEND_URL;
 		}
+
+		// Wrap each API request in a CLIENT span and inject its W3C trace headers, so backend spans correlate with the SPA
+		this.api.interceptors.request.use((config) => {
+			const method = (config.method ?? 'get').toUpperCase();
+			const url = `${config.baseURL ?? ''}${config.url ?? ''}`;
+			// startRequestSpan returns undefined when tracing is disabled, so only track a span when one was created.
+			const span = startRequestSpan(method, url, config.headers);
+			if (span) {
+				requestSpans.set(config, span);
+			}
+			return config;
+		});
+		this.api.interceptors.response.use(
+			(response) => {
+				endRequestSpan(requestSpans.get(response.config), response.status);
+				requestSpans.delete(response.config);
+				return response;
+			},
+			(error: unknown) => {
+				if (error instanceof AxiosError && error.config) {
+					endRequestSpan(requestSpans.get(error.config), error.response?.status, error);
+					requestSpans.delete(error.config);
+				}
+				return Promise.reject(error);
+			}
+		);
 	}
 }
 

frontend/src/lib/types/application-configuration.type.ts

@@ -13,6 +13,7 @@ export type AppConfig = {
 	uiConfigDisabled: boolean;
 	accentColor: string;
 	requireUserEmail: boolean;
+	tracingEnabled: boolean;
 };
 
 export type AllAppConfig = AppConfig & {

frontend/src/lib/utils/tracing-util.ts

@@ -0,0 +1,183 @@
+// Frontend tracing built on the OpenTelemetry Web SDK.
+//
+// A page-level span is started for each page view and a CLIENT span is created for every API request as its child, with the W3C `traceparent` header injected into the outgoing request.
+// The backend server spans therefore join the same trace, so a page view and everything it triggers — frontend, backend and database — show up as one trace rooted at the page view.
+//
+// Spans are exported over OTLP/HTTP to a same-origin backend endpoint (`/internal/telemetry/traces`), which forwards them to the collector the backend already uses.
+// Same-origin means no CORS and no browser-facing collector to configure.
+//
+// The OpenTelemetry libraries are heavy, so they are only pulled in via dynamic `import()` once tracing is
+// enabled (see `setTracingEnabled`). When tracing is disabled they are never loaded and stay out of the main app bundle.
+
+import { browser, version } from '$app/environment';
+// Type-only imports are erased at build time, so they don't pull the OpenTelemetry libraries into the bundle.
+import type { Context, Span } from '@opentelemetry/api';
+
+const TRACER_NAME = 'pocket-id-frontend';
+
+// Same-origin endpoint that forwards the browser's OTLP payloads to the backend's collector.
+const EXPORTER_URL = '/internal/telemetry/traces';
+
+/** Minimal shape of a header carrier we can inject into (satisfied by axios' AxiosHeaders). */
+export type HeaderCarrier = { set(name: string, value: string): void };
+
+// Runtime bindings from `@opentelemetry/api`, loaded lazily via dynamic import once tracing is enabled.
+// Undefined until the OpenTelemetry libraries have been imported and the tracer provider registered.
+let otel: typeof import('@opentelemetry/api') | undefined;
+
+// Memoizes the dynamic import + provider setup so the libraries are loaded and the provider registered only once.
+let providerPromise: Promise<void> | undefined;
+
+// Whether the backend is configured to receive traces (OTEL_TRACES_EXPORTER=otlp with a resolvable collector endpoint).
+// It stays false until the app configuration is loaded, so nothing is exported to `/internal/telemetry/traces` when tracing is disabled (that route isn't even registered then).
+let tracingEnabled = false;
+
+// The current page-level span and a Context carrying it.
+// API request spans are created as its children so a page view groups everything it triggers into one trace.
+// `pageContext` is undefined while no page trace is active.
+let pageSpan: Span | undefined;
+let pageContext: Context | undefined;
+
+/**
+ * Enables or disables frontend tracing based on the backend configuration.
+ * Called from the root layout once the application configuration is loaded.
+ * When enabled, the OpenTelemetry libraries are dynamically imported and the tracer provider is set up;
+ * when disabled, they are never loaded, so nothing is sent to the backend telemetry endpoint and the libraries stay out of the main bundle.
+ */
+export async function setTracingEnabled(enabled: boolean): Promise<void> {
+	tracingEnabled = enabled;
+	if (enabled) {
+		await ensureProvider();
+	}
+}
+
+// Dynamically imports the OpenTelemetry libraries (so they're only fetched when tracing is enabled) and installs the tracer provider.
+// Safe to call multiple times: the work is memoized and only runs once.
+function ensureProvider(): Promise<void> {
+	if (!browser || !tracingEnabled) {
+		return Promise.resolve();
+	}
+	providerPromise ??= loadAndRegister();
+	return providerPromise;
+}
+
+async function loadAndRegister(): Promise<void> {
+	try {
+		const [
+			api,
+			{ OTLPTraceExporter },
+			{ resourceFromAttributes },
+			{ BatchSpanProcessor, WebTracerProvider },
+			{ ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION }
+		] = await Promise.all([
+			import('@opentelemetry/api'),
+			import('@opentelemetry/exporter-trace-otlp-http'),
+			import('@opentelemetry/resources'),
+			import('@opentelemetry/sdk-trace-web'),
+			import('@opentelemetry/semantic-conventions')
+		]);
+
+		const provider = new WebTracerProvider({
+			resource: resourceFromAttributes({
+				[ATTR_SERVICE_NAME]: TRACER_NAME,
+				[ATTR_SERVICE_VERSION]: version
+			}),
+			spanProcessors: [new BatchSpanProcessor(new OTLPTraceExporter({ url: EXPORTER_URL }))]
+		});
+
+		// register() installs the provider globally together with the default W3C trace context (`traceparent`) propagator.
+		provider.register();
+
+		// End (and flush) the current page span when the tab/page goes away, so its trace completes.
+		addEventListener('pagehide', () => {
+			endPageSpan();
+			void provider.forceFlush();
+		});
+
+		otel = api;
+	} catch (e) {
+		// Tracing is best-effort: never let a failure to load the OpenTelemetry libraries break the app.
+		console.error('Failed to initialize frontend tracing', e);
+	}
+}
+
+function endPageSpan(): void {
+	pageSpan?.end();
+	pageSpan = undefined;
+	pageContext = undefined;
+}
+
+/**
+ * Starts a new page-level trace, ending the previous one, so that a page view and the API calls it triggers form a single trace rooted at the page view.
+ * Called on each navigation from the root +layout.
+ */
+export function startPageTrace(path?: string): void {
+	if (!browser || !tracingEnabled || !otel) {
+		return;
+	}
+	endPageSpan();
+
+	pageSpan = otel.trace
+		.getTracer(TRACER_NAME)
+		.startSpan(path ? `pageview ${path}` : 'pageview', {
+			root: true,
+			kind: otel.SpanKind.INTERNAL
+		});
+	pageContext = otel.trace.setSpan(otel.ROOT_CONTEXT, pageSpan);
+}
+
+/**
+ * Creates a CLIENT span for an outgoing API request (as a child of the current page span) and injects the W3C trace headers into `carrier`.
+ * The returned span must be ended with  {@link endRequestSpan} once the response (or error) is available.
+ */
+export function startRequestSpan(
+	method: string,
+	url: string,
+	carrier: HeaderCarrier
+): Span | undefined {
+	if (!tracingEnabled || !otel) {
+		return undefined;
+	}
+
+	// If no navigation has happened yet (e.g. the initial page load), start the page trace lazily so the initial requests are grouped under it too.
+	if (!pageContext) {
+		startPageTrace(typeof location === 'undefined' ? undefined : location.pathname);
+	}
+
+	const parent = pageContext ?? otel.ROOT_CONTEXT;
+	const span = otel.trace.getTracer(TRACER_NAME).startSpan(
+		`${method} ${url}`,
+		{
+			kind: otel.SpanKind.CLIENT,
+			attributes: {
+				'http.request.method': method,
+				'url.full': url
+			}
+		},
+		parent
+	);
+
+	otel.propagation.inject(otel.trace.setSpan(parent, span), carrier, {
+		set: (c, key, value) => c.set(key, value)
+	});
+
+	return span;
+}
+
+/** Ends a request span, recording the response status (and error, if any). */
+export function endRequestSpan(span: Span | undefined, statusCode?: number, error?: unknown): void {
+	// A span can only exist when the OpenTelemetry libraries were loaded, so `otel` is defined here.
+	if (!span || !otel) {
+		return;
+	}
+	if (statusCode !== undefined) {
+		span.setAttribute('http.response.status_code', statusCode);
+	}
+	if (error) {
+		span.recordException(error instanceof Error ? error : String(error));
+		span.setStatus({ code: otel.SpanStatusCode.ERROR });
+	} else if (statusCode !== undefined && statusCode >= 400) {
+		span.setStatus({ code: otel.SpanStatusCode.ERROR });
+	}
+	span.end();
+}

frontend/src/routes/+layout.svelte

@@ -4,11 +4,16 @@
 	import Header from '$lib/components/header/header.svelte';
 	import { Toaster } from '$lib/components/ui/sonner';
 	import { m } from '$lib/paraglide/messages';
+	import { startPageTrace } from '$lib/utils/tracing-util';
+	import { beforeNavigate } from '$app/navigation';
 	import { ModeWatcher } from 'mode-watcher';
 	import { type Snippet } from 'svelte';
 	import '../app.css';
 	import type { LayoutData } from './$types';
 
+	// Start a new page-level trace on each navigation so a page view and the API calls it triggers are correlated as a single trace.
+	beforeNavigate((nav) => startPageTrace(nav.to?.url.pathname));
+
 	let {
 		data,
 		children

frontend/src/routes/+layout.ts

@@ -4,6 +4,7 @@ import appConfigStore from '$lib/stores/application-configuration-store';
 import userStore from '$lib/stores/user-store';
 import { setLocaleForLibraries } from '$lib/utils/locale.util';
 import { getAuthRedirectPath } from '$lib/utils/redirection-util';
+import { setTracingEnabled } from '$lib/utils/tracing-util';
 import { redirect } from '@sveltejs/kit';
 import type { LayoutLoad } from './$types';
 
@@ -37,6 +38,10 @@ export const load: LayoutLoad = async ({ url }) => {
 		appConfigStore.set(appConfig);
 	}
 
+	// Only export traces when the backend is configured to forward them; otherwise the SPA would POST spans to an unregistered endpoint.
+	// When enabled this dynamically imports the OpenTelemetry libraries, so awaiting it ensures the tracer provider is ready before any traced navigation or request.
+	await setTracingEnabled(appConfig?.tracingEnabled ?? false);
+
 	await setLocaleForLibraries();
 
 	return {

frontend/vite.config.ts

@@ -39,6 +39,9 @@ export default defineConfig((mode) => {
 				'/api': {
 					target: process.env.DEVELOPMENT_BACKEND_URL || 'http://localhost:1411'
 				},
+				'/internal': {
+					target: process.env.DEVELOPMENT_BACKEND_URL || 'http://localhost:1411'
+				},
 				'/.well-known': {
 					target: process.env.DEVELOPMENT_BACKEND_URL || 'http://localhost:1411'
 				},