Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ export enum RoutePaths {
// Trace extensions
TRACE_SPANS_FILTERS = "/opsml/api/scouter/trace/spans/filters",
TRACE_FACETS = "/opsml/api/scouter/trace/facets",
TRACE_SPANS_BY_ID = "/opsml/api/scouter/trace",
PROFILES_LIST = "/opsml/api/scouter/profiles",
// GenAI
GENAI_TOKEN_METRICS = "/opsml/api/scouter/genai/metrics/tokens",
Expand Down Expand Up @@ -138,6 +139,7 @@ export enum ServerPaths {
TRACE_METRICS = "/api/scouter/observability/trace/metrics",
TRACE_SPANS = "/api/scouter/observability/trace/spans",
TRACE_PAGE = "/api/scouter/observability/trace",
TRACE_SPANS_BY_ID = "/api/scouter/observability/trace",
ENTITY_ID_TAGS = "/api/scouter/tags/entity",
// GenAI
GENAI_TOKEN_METRICS = "/api/scouter/genai/metrics/tokens",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@
// Working mutable state (refreshed on time range changes)
let evalData = $state<AgentPromptEvalData[]>(agentPromptEvals);
let isRefreshing = $state(false);
// Snapshot on mount so stale singleton state from prior navigation never triggers an immediate refresh.
let lastSeenRange = $state(timeRangeState.selectedTimeRange);
let lastSeenSignal = $state(timeRangeState.refreshSignal);

// ── Time Range Refresh ────────────────────────────────────────────────────────
Expand All @@ -60,13 +62,12 @@
const signalFired = signal !== lastSeenSignal;
if (!newRange) return;

const rangeChanged = evalData.some(e => {
if (e.monitoringData.status !== 'success') return false;
const r = e.monitoringData.selectedTimeRange;
return r.startTime !== newRange.startTime || r.endTime !== newRange.endTime;
});
const rangeChanged =
lastSeenRange.startTime !== newRange.startTime ||
lastSeenRange.endTime !== newRange.endTime;

if (rangeChanged || signalFired) {
lastSeenRange = newRange;
lastSeenSignal = signal;
evalData.forEach(e => {
if (e.monitoringData.status === 'success') {
Expand All @@ -80,6 +81,7 @@
/** Full refresh without cursors — resets pagination to page 1. */
async function performRefresh() {
isRefreshing = true;
timeRangeState.beginRefresh();
try {
await Promise.all(
evalData.map(async (e) => {
Expand All @@ -91,12 +93,14 @@
console.error('[AgentEvalDashboard] Refresh failed:', err);
} finally {
isRefreshing = false;
timeRangeState.endRefresh();
}
}

/** Advance the record page for all evals that have a cursor in that direction. */
async function handleRecordPageChange(direction: string) {
isRefreshing = true;
timeRangeState.beginRefresh();
try {
await Promise.all(
evalData.map(async (e) => {
Expand All @@ -114,12 +118,14 @@
console.error('[AgentEvalDashboard] Record page change failed:', err);
} finally {
isRefreshing = false;
timeRangeState.endRefresh();
}
}

/** Advance the workflow page for all evals that have a cursor in that direction. */
async function handleWorkflowPageChange(direction: string) {
isRefreshing = true;
timeRangeState.beginRefresh();
try {
await Promise.all(
evalData.map(async (e) => {
Expand All @@ -137,6 +143,7 @@
console.error('[AgentEvalDashboard] Workflow page change failed:', err);
} finally {
isRefreshing = false;
timeRangeState.endRefresh();
}
}

Expand Down Expand Up @@ -283,6 +290,14 @@
/** Merged workflow page: items from all evals sorted by created_at desc. */
const workflowPage = $derived(
(() => {
const recordTraceMap = new Map<string, string>();
evalData.forEach(e => {
if (e.monitoringData.status !== 'success') return;
e.monitoringData.selectedData.records?.items?.forEach(r => {
if (r.trace_id) recordTraceMap.set(r.uid, r.trace_id);
});
});

const items: WorkflowWithAgent[] = [];
let hasNext = false;
let hasPrevious = false;
Expand All @@ -291,7 +306,13 @@
const path = promptEvalPath(e);
const profile = e.monitoringData.profile as AgentEvalProfile;
e.monitoringData.selectedData.workflows?.items?.forEach(w =>
items.push({ ...w, _agentName: e.promptCard.name, _evalPath: path, _profile: profile })
items.push({
...w,
_agentName: e.promptCard.name,
_evalPath: path,
_profile: profile,
_traceId: recordTraceMap.get(w.record_uid),
Comment on lines 306 to +314

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 _traceId lookup fails when records and workflows are on different pages

The workflowPage derived block builds a recordTraceMap from the current page of records (e.monitoringData.selectedData.records?.items), then sets _traceId: recordTraceMap.get(w.record_uid) for each workflow. Since records and workflows are paginated independently via handleRecordPageChange and handleWorkflowPageChange (crates/opsml_server/opsml_ui/src/lib/components/card/agent/evaluation/AgentEvalDashboard.svelte:98-122), after the user pages forward on either table, the map only contains UIDs from the current record page. Any workflow referencing a record on a different page will have _traceId set to undefined, silently hiding the trace link even when the record actually has a trace_id.

(Refers to lines 293-314)

Prompt for agents
The _traceId for each workflow is looked up from a map built from only the currently-visible page of records. Since records and workflows paginate independently (handleRecordPageChange vs handleWorkflowPageChange), a workflow on page N may reference a record_uid that is on a different records page, causing the map lookup to return undefined and the trace link to disappear.

Possible approaches:
1. Maintain a cumulative record trace map (Map<string, string>) across all pages ever loaded, not just the current page. Append to it in the effect whenever new records arrive.
2. Include trace_id directly on the AgentEvalWorkflowResult type from the backend, so workflows carry their own trace_id without needing to cross-reference records.
3. If the relationship is always 1:1, denormalize by having the backend embed trace_id in the workflow response.

Files involved: AgentEvalDashboard.svelte (workflowPage derived block), types.ts (WorkflowWithAgent), and potentially the backend workflow endpoint.
Open in Devin Review

Was this helpful? React with 👍 or 👎 to provide feedback.

})
);
if (e.monitoringData.selectedData.workflows?.has_next) hasNext = true;
if (e.monitoringData.selectedData.workflows?.has_previous) hasPrevious = true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,7 @@
<AgentEvalWorkflowSideBar
selectedWorkflow={selectedWorkflow}
profile={selectedWorkflow._profile}
traceId={selectedWorkflow._traceId}
onClose={handleClosePanel}
/>
{/if}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ export type WorkflowWithAgent = AgentEvalWorkflowResult & {
_agentName: string;
_evalPath: string;
_profile: AgentEvalProfile;
_traceId?: string;
};

/** Merged pagination state for the agent eval record table. */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ export function isPromptCard(obj: unknown): obj is PromptCard {

const card = obj as Partial<PromptCard>;
return (
card.registry_type === RegistryType.Prompt &&
String(card.registry_type).toLowerCase() === RegistryType.Prompt &&
typeof card.prompt === "object" &&
card.prompt !== null &&
typeof card.name === "string" &&
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
import { refreshAgentMonitoringData } from '$lib/components/scouter/dashboard/utils';
import type { RecordCursor } from '$lib/components/scouter/types';
import { getMaxDataPoints, type RegistryType } from '$lib/utils';
import { Loader2 } from 'lucide-svelte';
import AgentDashboard from '$lib/components/scouter/agent/dashboard/AgentDashboard.svelte';
import AgentTaskAccordion from '$lib/components/scouter/agent/task/AgentTaskAccordion.svelte';
import { timeRangeState } from '$lib/components/utils/timeState.svelte';
Expand All @@ -29,19 +28,21 @@
let monitoringData = $state<AgentMonitoringPageData>(initialMonitoringData);
let isRefreshing = $state(false);
let currentMaxPoints = $state(typeof window !== 'undefined' ? getMaxDataPoints() : 0);
// Snapshot on mount so stale singleton state from prior navigation never triggers an immediate refresh.
let lastSeenRange = $state(timeRangeState.selectedTimeRange);
let lastSeenSignal = $state(timeRangeState.refreshSignal);

$effect(() => {
if (isRefreshing) return;
const newRange = timeRangeState.selectedTimeRange;
const signal = timeRangeState.refreshSignal;
if (newRange && monitoringData?.status === 'success') {
const currentRange = monitoringData.selectedTimeRange;
const rangeChanged =
currentRange.startTime !== newRange.startTime ||
currentRange.endTime !== newRange.endTime;
lastSeenRange.startTime !== newRange.startTime ||
lastSeenRange.endTime !== newRange.endTime;
const signalFired = signal !== lastSeenSignal;
if (rangeChanged || signalFired) {
lastSeenRange = newRange;
lastSeenSignal = signal;
monitoringData.selectedTimeRange = newRange;
performRefresh();
Expand Down Expand Up @@ -72,6 +73,7 @@
) {
if (!monitoringData || monitoringData.status !== 'success') return;
isRefreshing = true;
timeRangeState.beginRefresh();
try {
await refreshAgentMonitoringData(fetch, monitoringData, {
recordCursor: rCursor,
Expand All @@ -81,6 +83,7 @@
console.error('Agent Dashboard Refresh Failed', e);
} finally {
isRefreshing = false;
timeRangeState.endRefresh();
}
}

Expand All @@ -93,14 +96,6 @@
}
</script>

{#if isRefreshing}
<div class="fixed top-4 right-4 z-50 flex items-center gap-2 px-4 py-2 bg-black text-white
rounded-lg shadow-lg animate-pulse border-2 border-white transition-opacity duration-200">
<Loader2 class="w-4 h-4 animate-spin" />
<span class="text-xs font-bold uppercase tracking-wider">Syncing...</span>
</div>
{/if}

{#if monitoringData.status === 'error'}
<AgentTaskAccordion tasks={monitoringData.profile.tasks} />
<MonitoringErrorView
Expand All @@ -112,7 +107,7 @@
{registryType}
/>
{:else if monitoringData.status === 'success'}
<div class="transition-opacity duration-200 {isRefreshing ? 'opacity-60 pointer-events-none grayscale-[0.5]' : ''}">
<div class="transition-opacity duration-200 {isRefreshing ? 'opacity-60 grayscale-[0.5]' : ''}">
<AgentDashboard
bind:monitoringData
onRecordPageChange={handleRecordPageChange}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
<script lang="ts">
import { X, FileJson, AlertCircle, Clock, Tag, ChevronDown, ChevronUp } from 'lucide-svelte';
import { X, FileJson, AlertCircle, Clock, Tag, ChevronDown, ChevronUp, Activity } from 'lucide-svelte';
import { page } from '$app/state';
import type { EvalRecord } from '../types';
import { Status } from '../types';
import CodeBlock from '$lib/components/codeblock/CodeBlock.svelte';
Expand All @@ -16,6 +17,10 @@

let contextOpen = $state(true);

const observabilityPath = $derived(
record.trace_id ? page.url.pathname.replace(/\/evaluation(\/.*)?$/, '/observability') : null
);

function getStatusBadgeClass(status: Status): string {
switch (status) {
case Status.Processed: return 'bg-secondary-100 text-secondary-900 border-black';
Expand Down Expand Up @@ -125,6 +130,16 @@
{formatDuration(record.processing_duration)}
</span>
{/if}
{#if record.trace_id && observabilityPath}
<a
href="{observabilityPath}?trace_id={record.trace_id}"
class="inline-flex items-center gap-1 px-2 py-0.5 text-xs font-bold border-2 border-black bg-purple-100 text-purple-900 rounded-base shadow-small hover:bg-purple-200 transition-colors duration-100"
title="Open trace in Observability"
>
<Activity class="w-3 h-3" />
Trace
</a>
{/if}
{#if showCloseButton && onClose}
<button
onclick={onClose}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,17 @@
actual: unknown;
}>();

const isComplex = $derived(
const isStacked = $derived(
(typeof expected === 'object' && expected !== null) ||
(typeof actual === 'object' && actual !== null)
(typeof actual === 'object' && actual !== null) ||
typeof expected === 'string' ||
typeof actual === 'string'
);

function isObject(val: unknown): val is object {
return typeof val === 'object' && val !== null;
}

function getTypeHint(val: unknown): string {
if (val === null || val === undefined) return 'null';
if (Array.isArray(val)) return `array - ${val.length} items`;
Expand All @@ -26,7 +32,7 @@
}
</script>

{#if !isComplex}
{#if !isStacked}
<div class="grid grid-cols-2 gap-3">
<div class="flex flex-col gap-1.5">
<div class="flex items-center gap-2">
Expand Down Expand Up @@ -55,15 +61,21 @@
<span class="text-xs font-black uppercase tracking-wide text-primary-700">Expected</span>
<span class="text-xs font-mono text-primary-500">{getTypeHint(expected)}</span>
</div>
<div class="bg-surface-50 rounded-base border-2 border-black p-1 shadow-small text-xs overflow-hidden">
<CodeBlock
code={formatValue(expected)}
showLineNumbers={false}
lang="json"
prePadding="p-1"
classes="h-full"
/>
</div>
{#if isObject(expected)}
<div class="bg-surface-50 rounded-base border-2 border-black p-1 shadow-small text-xs overflow-hidden">
<CodeBlock
code={formatValue(expected)}
showLineNumbers={false}
lang="json"
prePadding="p-1"
classes="h-full"
/>
</div>
{:else}
<div class="bg-surface-50 rounded-base border-2 border-black p-3 shadow-small">
<span class="text-sm font-mono text-primary-950 whitespace-pre-wrap break-words">{formatValue(expected)}</span>
</div>
{/if}
</div>

<div class="border-t-2 border-black/10"></div>
Expand All @@ -73,15 +85,21 @@
<span class="text-xs font-black uppercase tracking-wide text-primary-700">Actual</span>
<span class="text-xs font-mono text-primary-500">{getTypeHint(actual)}</span>
</div>
<div class="bg-surface-50 rounded-base border-2 border-black p-1 shadow-small text-xs overflow-hidden">
<CodeBlock
code={formatValue(actual)}
showLineNumbers={false}
lang="json"
prePadding="p-1"
classes="h-full"
/>
</div>
{#if isObject(actual)}
<div class="bg-surface-50 rounded-base border-2 border-black p-1 shadow-small text-xs overflow-hidden">
<CodeBlock
code={formatValue(actual)}
showLineNumbers={false}
lang="json"
prePadding="p-1"
classes="h-full"
/>
</div>
{:else}
<div class="bg-surface-50 rounded-base border-2 border-black p-3 shadow-small">
<span class="text-sm font-mono text-primary-950 whitespace-pre-wrap break-words">{formatValue(actual)}</span>
</div>
{/if}
</div>
</div>
{/if}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
type TraceAssertion,
} from '../task';
import { Accordion } from '@skeletonlabs/skeleton-svelte';
import { Info, Activity, AlertCircle, GitBranch, CheckCircle2, XCircle, TrendingUp, MessageSquareText, ChevronDown } from 'lucide-svelte';
import { Info, Activity, AlertCircle, GitBranch, CheckCircle2, XCircle, TrendingUp, MessageSquareText, ChevronDown, ExternalLink } from 'lucide-svelte';
import Pill from '$lib/components/utils/Pill.svelte';
import ComparisonView from '$lib/components/scouter/agent/task/ComparisonView.svelte';
import TraceAssertionPill from './TraceAssertionPill.svelte';
Expand All @@ -17,9 +17,11 @@
import { AgentEvalProfileHelper } from '../utils';
import PromptModal from '$lib/components/card/prompt/common/PromptModal.svelte';

let { task, profile } = $props<{
let { task, profile, traceId, observabilityPath } = $props<{
task: EvalTaskResult;
profile: AgentEvalProfile;
traceId?: string;
observabilityPath?: string | null;
}>();

const active_task: EvalTaskResult = $derived(task);
Expand Down Expand Up @@ -158,6 +160,16 @@
/>
<Pill key="Duration" value={durationStr} textSize="text-xs" />
<Pill key="Score" value={active_task.value.toFixed(4)} textSize="text-xs" />
{#if traceId && observabilityPath}
<a
href="{observabilityPath}?trace_id={traceId}"
class="inline-flex items-center gap-1 px-2 py-0.5 text-xs font-bold border-2 border-black bg-surface-50 text-primary-800 rounded-base shadow-small shadow-click-small"
title="Open trace in Observability"
>
<ExternalLink class="w-3 h-3" />
Trace
</a>
{/if}
</div>
</div>
<div class="mr-2 mt-1 flex-shrink-0">
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ export interface EvalRecord {
entity_uid: string;
status: Status;
entity_type: EntityType;
trace_id?: string;
}

/**
Expand Down
Loading
Loading