diff --git a/examples/metrics/metrics_configs_sglang.yaml b/examples/metrics/metrics_configs_sglang.yaml new file mode 100644 index 000000000..10e3cb450 --- /dev/null +++ b/examples/metrics/metrics_configs_sglang.yaml @@ -0,0 +1,1095 @@ +# SGLang UCM metrics: shared by HiCache and direct Linker integrations. +# The adapter records physical storage transfers, not inference batches or queue submissions. +# Use SGLang's PROMETHEUS_MULTIPROC_DIR; do not configure a separate directory here. +log_interval: 5 +multiproc_prefix: "ucm:" +consumers: + multiproc: true + vllm_connector: false + +# ============================================================================ +# Common bucket aliases (for readability only - YAML doesn't inline these). +# Durations are in ms except KV metrics, which use seconds; bandwidth in GB/s. +# ============================================================================ + +# Counter metrics configuration +counter: + - name: "sglang_connector_load_requests_total" + documentation: "SGLang connector load: Physical storage transfer calls attempted, per worker and pool or layer" + - name: "sglang_connector_load_bytes_total" + documentation: "SGLang connector load: Bytes transferred by successful storage calls, local to this worker" + - name: "sglang_connector_load_submit_errors_total" + documentation: "SGLang connector load: Storage submission failures" + - name: "sglang_connector_load_wait_errors_total" + documentation: "SGLang connector load: Storage completion wait failures" + - name: "sglang_connector_dump_requests_total" + documentation: "SGLang connector dump: Physical storage transfer calls attempted, per worker and pool or layer" + - name: "sglang_connector_dump_bytes_total" + documentation: "SGLang connector dump: Bytes transferred by successful storage calls, local to this worker" + - name: "sglang_connector_dump_submit_errors_total" + documentation: "SGLang connector dump: Storage submission failures" + - name: "sglang_connector_dump_wait_errors_total" + documentation: "SGLang connector dump: Storage completion wait failures" + # BEGIN GENERATED KV counter + # -- AsuStore observability ------------------------------------------------ + - name: "asu_lookup_requests_total" + documentation: "AsuStore Lookup, prefix lookup and reverse lookup calls" + - name: "asu_lookup_errors_total" + documentation: "Failed AsuStore lookup calls; excludes timeout fallback to misses" + - name: "asu_lookup_query_blocks_total" + documentation: "Blocks queried by normally returned Lookup calls, including timeout fallback" + - name: "asu_lookup_hit_blocks_total" + documentation: "Hit blocks returned by Lookup; timeout fallback contributes zero hits" + - name: "asu_lookup_prefix_query_blocks_total" + documentation: "Blocks queried by normally returned prefix lookup calls, including timeout fallback" + - name: "asu_lookup_prefix_hit_blocks_total" + documentation: "Contiguous prefix hit blocks, computed as returned index + 1; timeout fallback contributes zero hits" + - name: "asu_lookup_timeout_fallback_total" + documentation: "AsuStore lookup calls returning cache misses after query timeout" + - name: "asu_load_requests_total" + documentation: "AsuStore Load calls, including validation and submission failures" + - name: "asu_load_submit_errors_total" + documentation: "Failed AsuStore Load submissions, including buffer validation failures" + - name: "asu_load_submitted_bytes_total" + documentation: "Payload bytes accepted by AsuStore Load; not successfully completed bytes" + - name: "asu_dump_requests_total" + documentation: "AsuStore Dump calls, including validation and submission failures" + - name: "asu_dump_submit_errors_total" + documentation: "Failed AsuStore Dump submissions, including buffer validation failures" + - name: "asu_dump_submitted_bytes_total" + documentation: "Payload bytes accepted by AsuStore Dump; not successfully completed bytes" + - name: "asu_wait_requests_total" + documentation: "AsuStore Wait calls; repeated waits are counted separately" + - name: "asu_wait_errors_total" + documentation: "AsuStore Wait calls returning errors, including timeouts and task failures" + - name: "asu_wait_timeouts_total" + documentation: "AsuStore Wait calls returning timeout; a subset of wait errors" + + - name: "kv_client_query_requests_total" + documentation: "Total KV client query submissions" + - name: "kv_client_query_entries_total" + documentation: "Total keys submitted to KV client query" + - name: "kv_client_query_errors_total" + documentation: "Total failed KV client query submissions" + - name: "kv_client_load_requests_total" + documentation: "Total KV client load API calls across single and batch protocols" + - name: "kv_client_load_entries_total" + documentation: "Total entries in KV client load API calls across single and batch protocols" + - name: "kv_client_load_errors_total" + documentation: "Total failed KV client load API submissions, excluding asynchronous execution failures" + - name: "kv_client_store_requests_total" + documentation: "Total KV client store API calls across single and batch protocols" + - name: "kv_client_store_entries_total" + documentation: "Total entries in KV client store API calls across single and batch protocols" + - name: "kv_client_store_errors_total" + documentation: "Total failed KV client store API submissions, excluding asynchronous execution failures" + - name: "kv_client_delete_requests_total" + documentation: "Total KV client delete submissions" + - name: "kv_client_delete_entries_total" + documentation: "Total keys submitted to KV client delete" + - name: "kv_client_delete_errors_total" + documentation: "Total failed KV client delete submissions" + - name: "kv_client_wait_requests_total" + documentation: "Total KV client wait calls" + - name: "kv_client_wait_errors_total" + documentation: "Total failed KV client wait calls" + - name: "kv_transport_task_completion_timeouts_total" + documentation: "Total KV transport task completion timeouts" + - name: "kv_transport_task_io_timeouts_total" + documentation: "Total KV transport sub-batch IO timeouts" + - name: "kv_transport_task_connection_errors_total" + documentation: "Total KV transport connection errors" + # END GENERATED KV counter + # -- Pipeline Cache stage: lookup hit/miss breakdown ---------------------- + # Hit counters let Prometheus compute hit rate without losing precision: + # rate(ucm:cache_lookup_hit_blocks_total[5m]) + # / (rate(hit) + rate(miss)) + - name: "cache_lookup_hit_blocks_total" + documentation: "Number of lookup hits served by the Cache stage (no descent to backend)" + - name: "cache_lookup_miss_blocks_total" + documentation: "Number of lookup misses at the Cache stage (had to query the backend)" + - name: "cache_load_shards_total" + documentation: "Total shards whose Cache buffer state was inspected during load" + - name: "cache_load_wait_shards_total" + documentation: "Shards whose Cache buffer was not ready when acquired and required waiting" + - name: "cache_load_backend_shards_total" + documentation: "Shards that descended to the backend on load (true cache miss at the buffer-allocation stage; aka backend-load count)" + - name: "cache_load_success_shards_total" + documentation: "Shards successfully loaded from an already-ready Cache buffer to device" + - name: "cache_posix_load_success_shards_total" + documentation: "Shards successfully loaded to device after waiting for Posix to fill Cache" + - name: "cache_load_failed_shards_total" + documentation: "Cache load shards that did not complete device delivery" + - name: "cache_dump_shards_total" + documentation: "Total shard descriptors processed by Cache dump, including failed tasks" + - name: "cache_dump_backend_shards_total" + documentation: "Shards actually pushed to backend on dump (excludes !handle.Owner() skips in shared-buffer scenario)" + - name: "cache_load_queue_full_total" + documentation: "Number of Cache load submissions rejected because the waiting queue was full" + - name: "cache_dump_queue_full_total" + documentation: "Number of Cache dump submissions rejected because the waiting queue was full" + - name: "cache_backend_load_submit_errors_total" + documentation: "Number of Cache load backend submit failures" + - name: "cache_backend_load_wait_errors_total" + documentation: "Number of Cache load backend wait failures" + - name: "cache_backend_dump_submit_errors_total" + documentation: "Number of Cache dump backend submit failures" + - name: "cache_backend_dump_wait_errors_total" + documentation: "Number of Cache dump backend wait failures" + - name: "cache_h2d_errors_total" + documentation: "Number of Cache host-to-device transfer or sync failures" + - name: "cache_d2h_errors_total" + documentation: "Number of Cache device-to-host transfer, event wait, or sync failures" + + # -- Pipeline Cache stage: per-worker real throughput counters ------------ + # Accumulated per completed Cache task with size = shardSize * num_shards. + # Use rate() / 1e9 to get actual GB/s the worker is moving through the + # Cache stage — aggregates across concurrent tasks and includes idle gaps, + # which the per-task cache_*_bandwidth_gbps histogram cannot reveal. + - name: "cache_load_bytes_total" + documentation: "Total bytes loaded through the Cache stage (per-task size summed)" + - name: "cache_dump_bytes_total" + documentation: "Total bytes dumped through the Cache stage (per-task size summed)" + + # -- Pipeline Posix stage: real per-worker throughput counters ------------ + # Accumulated per completed Posix task. Use + # rate() / 1e9 to get actual GB/s the worker is moving across the disk, + # which the per-task *_bandwidth_gbps histogram cannot reveal because + # it does not aggregate concurrent IO threads or account for idle gaps. + - name: "posix_s2h_bytes_total" + documentation: "Total bytes transferred from posix storage to host buffer (load path, summed per completed task)" + - name: "posix_h2s_bytes_total" + documentation: "Total bytes transferred from host buffer to posix storage (dump path, summed per completed task)" + - name: "posix_lookup_query_blocks_total" + documentation: "Total blocks submitted to Posix lookup" + - name: "posix_lookup_hit_blocks_total" + documentation: "Blocks found by Posix lookup" + - name: "posix_healthy_count_total" + documentation: "Number of successful Posix health probes" + - name: "posix_unhealthy_count_total" + documentation: "Number of failed Posix health probes" + - name: "posix_passive_failures_total" + documentation: "Number of failed Posix IO tasks observed by passive health detection" + - name: "posix_aio_timeout_total" + documentation: "Number of Posix AIO task or submit timeouts" + - name: "posix_io_timeout_total" + documentation: "Number of Posix synchronous worker task timeouts" + - name: "posix_open_errors_total" + documentation: "Number of Posix open failures" + - name: "posix_io_errors_total" + documentation: "Number of Posix read, write, or AIO completion failures" + + # -- YuanRong stage: successful shard source and resource-log counters ---- + - name: posix_handle_cache_hit_total + documentation: Number of Posix load handle-cache hits + - name: posix_handle_cache_miss_total + documentation: Number of Posix load handle-cache misses + - name: posix_handle_cache_evict_total + documentation: Number of Posix load handle-cache CLOCK evictions + - name: posix_handle_cache_bypass_total + documentation: Number of Posix load opens that skipped the handle cache + - name: "yuanrong_load_success_shards_total" + documentation: "Shards successfully loaded from YuanRong to device" + - name: "yuanrong_lookup_miss_posix_load_success_shards_total" + documentation: "Shards successfully loaded from Posix after YuanRong lookup miss" + - name: "yuanrong_load_fallback_posix_load_success_shards_total" + documentation: "Shards successfully loaded from Posix after YuanRong load failure" + - name: "yuanrong_load_failed_shards_total" + documentation: "YuanRong pipeline load shards that did not complete device delivery" + - name: "yuanrong_local_dram_load_hits_total" + documentation: "Estimated YuanRong local DRAM Get hits forwarded from kv_resource.log" + - name: "yuanrong_remote_load_hits_total" + documentation: "Estimated YuanRong remote worker Get hits forwarded from kv_resource.log" + - name: "yuanrong_local_ssd_load_hits_total" + documentation: "Estimated YuanRong local spill SSD Get hits forwarded from kv_resource.log" + - name: "yuanrong_l2_load_hits_total" + documentation: "YuanRong L2 persistence Get hits forwarded from kv_resource.log" + - name: "yuanrong_resource_log_read_errors_total" + documentation: "Number of failures opening, reading, or parsing YuanRong kv_resource.log" + + # -- Pipeline Mooncake stage --------------------------------------------- + - name: "mooncake_load_blocks_total" + documentation: "Total blocks loaded through the Mooncake stage" + - name: "mooncake_dump_blocks_total" + documentation: "Total blocks dumped through the Mooncake stage" + - name: "mooncake_lookup_hit_blocks_total" + documentation: "Blocks found directly by Mooncake lookup before backend descent" + - name: "mooncake_healthy_count_total" + documentation: "Number of successful Mooncake health probes" + - name: "mooncake_unhealthy_count_total" + documentation: "Number of failed Mooncake health probes" + - name: "mooncake_passive_failures_total" + documentation: "Number of failed Mooncake IO tasks observed by passive health detection" + - name: "mooncake_load_bytes_total" + documentation: "Total bytes loaded through the Mooncake stage" + - name: "mooncake_dump_bytes_total" + documentation: "Total bytes dumped through the Mooncake stage" + - name: "mooncake_load_hit_shards_total" + documentation: "Mooncake load shards served directly from Mooncake" + - name: "mooncake_load_miss_shards_total" + documentation: "Mooncake load shards that missed and descended to backend or recompute" + - name: "mooncake_load_backend_shards_total" + documentation: "Mooncake load shards submitted to the backend after a Mooncake miss" + - name: "mooncake_dump_existing_shards_total" + documentation: "Mooncake dump shards already present in Mooncake" + - name: "mooncake_dump_missing_shards_total" + documentation: "Mooncake dump shards written to Mooncake because they were missing" + - name: "mooncake_dump_backend_shards_total" + documentation: "Mooncake dump shards archived to the backend" + - name: "mooncake_load_queue_full_total" + documentation: "Number of Mooncake load submissions rejected because the waiting queue was full" + - name: "mooncake_dump_queue_full_total" + documentation: "Number of Mooncake dump submissions rejected because the waiting queue was full" + - name: "mooncake_get_errors_total" + documentation: "Number of Mooncake batch get failures" + - name: "mooncake_put_errors_total" + documentation: "Number of Mooncake batch put failures" + - name: "mooncake_backend_load_submit_errors_total" + documentation: "Number of Mooncake backend load submit failures" + - name: "mooncake_backend_load_wait_errors_total" + documentation: "Number of Mooncake backend load wait failures" + - name: "mooncake_backend_dump_submit_errors_total" + documentation: "Number of Mooncake backend dump submit failures" + - name: "mooncake_backend_dump_wait_errors_total" + documentation: "Number of Mooncake backend dump wait failures" + - name: "mooncake_h2d_errors_total" + documentation: "Number of Mooncake H2D transfer or sync failures" + - name: "mooncake_d2h_errors_total" + documentation: "Number of Mooncake D2H transfer, event wait, or sync failures" + - name: "mooncake_h2d_bytes_total" + documentation: "Total Mooncake bytes copied from host to device" + - name: "mooncake_d2h_bytes_total" + documentation: "Total Mooncake bytes copied from device to host" + + # -- Connector layer: per-worker real throughput counters ---------------- + - name: "load_bytes_total" + documentation: "Total bytes loaded through the UCM connector (summed across all start_load_kv calls)" + - name: "save_bytes_total" + documentation: "Total bytes saved through the UCM connector (summed across all wait_for_save calls)" + - name: "total_prefix_query_tokens_total" + documentation: "Total prefix cache query tokens observed by the UCM connector" + - name: "gpu_hbm_hit_tokens_total" + documentation: "Prefix cache tokens already hit in GPU or HBM before UCM lookup" + - name: "ucm_hit_tokens_total" + documentation: "Prefix cache tokens hit by the UCM connector" + - name: "total_prefix_query_blocks_total" + documentation: "Total full prefix blocks queried through the UCM connector" + - name: "gpu_hbm_hit_blocks_total" + documentation: "Full prefix blocks already hit in GPU or HBM before UCM lookup" + - name: "connector_lookup_errors_total" + documentation: "Number of connector lookup errors treated as cache misses" + - name: "connector_load_submit_errors_total" + documentation: "Number of connector load submit failures" + - name: "connector_load_wait_errors_total" + documentation: "Number of connector load wait failures" + - name: "connector_load_invalid_requests_total" + documentation: "Number of connector load failure events that invalidated request blocks" + - name: "connector_load_invalid_blocks_total" + documentation: "Number of newly invalidated vLLM block ids caused by connector load failures" + - name: "connector_dump_submit_errors_total" + documentation: "Number of connector dump submit failures" + - name: "connector_dump_wait_errors_total" + documentation: "Number of connector dump wait failures" + + # -- DramStore client observability ----------------------------------------- + - name: "dramstore_lookup_tasks_submitted_total" + documentation: "DramStore lookup: tasks submitted" + - name: "dramstore_lookup_tasks_rejected_total" + documentation: "DramStore lookup: tasks rejected" + - name: "dramstore_lookup_tasks_succeeded_total" + documentation: "DramStore lookup: tasks succeeded" + - name: "dramstore_lookup_tasks_failed_total" + documentation: "DramStore lookup: tasks failed" + - name: "dramstore_lookup_task_timeouts_total" + documentation: "DramStore lookup: task timeouts" + - name: "dramstore_lookup_requests_completed_total" + documentation: "DramStore lookup: requests completed" + - name: "dramstore_lookup_requests_failed_total" + documentation: "DramStore lookup: requests failed" + - name: "dramstore_lookup_request_timeouts_total" + documentation: "DramStore lookup: requests completed with Timeout status" + - name: "dramstore_dump_tasks_submitted_total" + documentation: "DramStore dump: tasks submitted" + - name: "dramstore_dump_tasks_rejected_total" + documentation: "DramStore dump: tasks rejected" + - name: "dramstore_dump_tasks_succeeded_total" + documentation: "DramStore dump: tasks succeeded" + - name: "dramstore_dump_tasks_failed_total" + documentation: "DramStore dump: tasks failed" + - name: "dramstore_dump_task_timeouts_total" + documentation: "DramStore dump: task timeouts" + - name: "dramstore_dump_requests_completed_total" + documentation: "DramStore dump: requests completed" + - name: "dramstore_dump_requests_failed_total" + documentation: "DramStore dump: requests failed" + - name: "dramstore_dump_request_timeouts_total" + documentation: "DramStore dump: requests completed with Timeout status" + - name: "dramstore_load_tasks_submitted_total" + documentation: "DramStore load: tasks submitted" + - name: "dramstore_load_tasks_rejected_total" + documentation: "DramStore load: tasks rejected" + - name: "dramstore_load_tasks_succeeded_total" + documentation: "DramStore load: tasks succeeded" + - name: "dramstore_load_tasks_failed_total" + documentation: "DramStore load: tasks failed" + - name: "dramstore_load_task_timeouts_total" + documentation: "DramStore load: task timeouts" + - name: "dramstore_load_requests_completed_total" + documentation: "DramStore load: requests completed" + - name: "dramstore_load_requests_failed_total" + documentation: "DramStore load: requests failed" + - name: "dramstore_load_request_timeouts_total" + documentation: "DramStore load: requests completed with Timeout status" + - name: "dramstore_connect_attempts_total" + documentation: "DramStore connect attempts" + - name: "dramstore_connect_failures_total" + documentation: "DramStore connect failures" + - name: "dramstore_fence_attempts_total" + documentation: "DramStore fence attempts" + - name: "dramstore_fence_failures_total" + documentation: "DramStore fence failures" + - name: "dramstore_fence_timeout_triggers_total" + documentation: "DramStore node fencing episodes triggered by request timeouts; excludes fence retries" + - name: "dramstore_reply_slot_nospace_total" + documentation: "DramStore reply slot acquisition failures due to exhausted capacity" + - name: "dramstore_stale_replies_total" + documentation: "DramStore stale replies" + + # -- DramPool server observability ------------------------------------------ + - name: "drampool_dump_requests_total" + documentation: "Total DUMP requests received and processed by DramPool" + - name: "drampool_load_requests_total" + documentation: "Total LOAD requests received and processed by DramPool" + - name: "drampool_lookup_requests_total" + documentation: "Total LOOKUP requests received and processed by DramPool" + - name: "drampool_dump_nospace_failures_total" + documentation: "DUMP entries whose buffer allocation finally failed with NoSpace after eviction retries (registration failures excluded)" + - name: "drampool_dump_failed_entries_total" + documentation: "Total DUMP entries that settled with a Failed result (all failure paths merged)" + - name: "drampool_load_miss_entries_total" + documentation: "LOAD entries that missed stored data (LoadBegin failure or request longer than stored length)" + - name: "drampool_lookup_miss_entries_total" + documentation: "LOOKUP entries that do not exist or are not READY" + - name: "drampool_transfer_failures_total" + documentation: "Data transfers that ended in a non-Completed terminal state (DUMP and LOAD merged)" + - name: "drampool_response_failures_total" + documentation: "Response return-path failures (local submit failures plus write-back transfer failures, DUMP and LOAD merged)" + - name: "drampool_submit_failures_total" + documentation: "Data transfer submission failures (ExecuteAsync failure or invalid handle)" + - name: "drampool_queue_request_full_total" + documentation: "requestQueue TryPush failures because the queue was full" + - name: "drampool_queue_completion_full_total" + documentation: "completionQueue full events that forced SubmitCompletion to spin-wait" + - name: "drampool_queue_response_buffer_retry_total" + documentation: "Response flag-buffer NoSpace events where SubmitResponse parked the request for retry" + + # -- DramPool resource reporter ----------------------------------------- + - name: "drampool_resource_log_read_errors_total" + documentation: "Number of failures opening, reading, or parsing the DramPool resource log" + +# Gauge metrics configuration + # Direct linker: per-worker counters and asynchronous operation timing. + - name: sglang_linker_lookup_requests_total + documentation: 'SGLang linker: Owner lookup attempts; excludes neutral non-owner responses' + - name: sglang_linker_lookup_errors_total + documentation: 'SGLang linker: Failed owner lookup attempts' + - name: sglang_linker_lookup_query_pages_total + documentation: 'SGLang linker: Full requested KV pages on successful owner lookups, including resumed + prefixes; repeated lookups count again' + - name: sglang_linker_lookup_hit_pages_total + documentation: 'SGLang linker: Maximum locally restorable KV page boundary on successful owner lookups; + before cross-rank intersection, not final token reuse' + - name: sglang_linker_lookup_resumed_total + documentation: 'SGLang linker: Owner lookup attempts with a previous hit boundary' + - name: sglang_linker_lookup_fallback_total + documentation: 'SGLang linker: Per-group prefix probes restarted at page zero after the cached boundary + disappeared' + - name: sglang_linker_lookup_skipped_total + documentation: 'SGLang linker: Neutral lookup responses on non-owner ranks; not actual storage hits' + - name: sglang_linker_load_requests_total + documentation: 'SGLang linker: Requests accepted into the scheduler''s pending load set' + - name: sglang_linker_load_cancelled_requests_total + documentation: 'SGLang linker: Pending requests removed by cancellation or reset before batching' + - name: sglang_linker_load_ready_layers_total + documentation: 'SGLang linker: Logical layers published ready across load batches; not physical Store + tasks' + - name: sglang_linker_offload_skipped_total + documentation: 'SGLang linker: Offload calls acknowledged on non-owner ranks without storage I/O' + - name: sglang_linker_load_check_errors_total + documentation: 'SGLang linker: Physical load tasks with a Check failure; also included in connector + wait errors when reaped' + - name: sglang_linker_dump_check_errors_total + documentation: 'SGLang linker: Physical dump tasks with a Check failure; also included in connector + wait errors when reaped' + - name: sglang_linker_load_operations_total + documentation: 'SGLang linker: Enqueued load batches' + - name: sglang_linker_load_started_total + documentation: 'SGLang linker: Dequeued load batches, before event synchronization and preparation' + - name: sglang_linker_load_completed_total + documentation: 'SGLang linker: Finished load batches, including failures; before ordered publication' + - name: sglang_linker_load_errors_total + documentation: 'SGLang linker: Failed load batches, including preparation, submission and completion + errors' + - name: sglang_linker_offload_operations_total + documentation: 'SGLang linker: Enqueued owner offload nodes' + - name: sglang_linker_offload_started_total + documentation: 'SGLang linker: Dequeued owner offload nodes, before event synchronization and preparation' + - name: sglang_linker_offload_completed_total + documentation: 'SGLang linker: Finished owner offload nodes, including failures; before ordered publication' + - name: sglang_linker_offload_errors_total + documentation: 'SGLang linker: Failed owner offload nodes, including preparation, submission and completion + errors' + +gauge: + - name: "yuanrong_dram_used_bytes" + documentation: "YuanRong physical shared-memory usage in bytes" + multiprocess_mode: "livemostrecent" + - name: "yuanrong_dram_capacity_bytes" + documentation: "YuanRong shared-memory capacity in bytes" + multiprocess_mode: "livemostrecent" + - name: "yuanrong_dram_usage_ratio" + documentation: "YuanRong physical shared-memory usage ratio" + multiprocess_mode: "livemostrecent" + - name: "yuanrong_ssd_used_bytes" + documentation: "YuanRong physical spill-disk usage in bytes" + multiprocess_mode: "livemostrecent" + - name: "yuanrong_ssd_capacity_bytes" + documentation: "YuanRong spill-disk capacity in bytes" + multiprocess_mode: "livemostrecent" + - name: "yuanrong_ssd_usage_ratio" + documentation: "YuanRong physical spill-disk usage ratio" + multiprocess_mode: "livemostrecent" + - name: "yuanrong_resource_log_last_update_timestamp_seconds" + documentation: "Unix timestamp of the latest YuanRong resource snapshot parsed by UCM" + multiprocess_mode: "livemostrecent" + - name: "yuanrong_resource_log_reporter_leader" + documentation: "Whether this UCM process is the host YuanRong resource reporter leader" + multiprocess_mode: "livemostrecent" + - name: "posix_store_used_bytes" + documentation: "Estimated logical Posix Store usage in bytes from GC sampling" + multiprocess_mode: "livemostrecent" + - name: "posix_store_capacity_bytes" + documentation: "Configured logical Posix Store capacity in bytes" + multiprocess_mode: "livemostrecent" + - name: "posix_store_usage_ratio" + documentation: "Estimated logical Posix Store usage ratio" + multiprocess_mode: "livemostrecent" + - name: "posix_store_health" + documentation: "Effective Posix health breaker state, where 1 is enabled and 0 is fused" + multiprocess_mode: "livemostrecent" + - name: "mooncake_store_health" + documentation: "Effective Mooncake health breaker state, where 1 is enabled and 0 is fused" + multiprocess_mode: "livemostrecent" + - name: "posix_gc_running" + documentation: "Posix garbage collection state, where 1 is running and 0 is idle" + multiprocess_mode: "livemostrecent" + + # -- DramStore client observability ----------------------------------------- + - name: "dramstore_scheduler_request_queue_size" + documentation: "Sampled queued Requests across scheduler runners, excluding batches already dequeued" + multiprocess_mode: livemostrecent + - name: "dramstore_scheduler_event_queue_size" + documentation: "Sampled queued NodeEvents across scheduler runners, excluding batches already dequeued" + multiprocess_mode: livemostrecent + - name: "dramstore_task_queue_size" + documentation: "Sampled TaskManager submission queue occupancy (tasks)" + multiprocess_mode: livemostrecent + - name: "dramstore_task_queue_capacity" + documentation: "TaskManager submission queue capacity (tasks)" + multiprocess_mode: livemostrecent + - name: "dramstore_completion_queue_size" + documentation: "Sampled TaskManager completion queue occupancy (request completions)" + multiprocess_mode: livemostrecent + - name: "dramstore_completion_queue_capacity" + documentation: "TaskManager completion queue capacity (request completions)" + multiprocess_mode: livemostrecent + - name: "dramstore_tasks_active" + documentation: "Sampled tasks awaiting completion, excluding queued submissions and retained results" + multiprocess_mode: livemostrecent + - name: "dramstore_io_entries_used" + documentation: "Sampled entries reserved by active tasks" + multiprocess_mode: livemostrecent + - name: "dramstore_io_entries_capacity" + documentation: "Maximum entries reserved by active tasks" + multiprocess_mode: livemostrecent + - name: "dramstore_reply_buffer_used_bytes" + documentation: "Sampled leased reply slot bytes including alignment, not payload bytes" + multiprocess_mode: livemostrecent + - name: "dramstore_reply_buffer_capacity_bytes" + documentation: "Total reply slot bytes including alignment" + multiprocess_mode: livemostrecent + - name: "dramstore_transport_queue_size" + documentation: "Sampled aggregate queued Transmit and Connect admission occupancy" + multiprocess_mode: livemostrecent + - name: "dramstore_transport_queue_capacity" + documentation: "Aggregate Transmit and Connect admission capacity" + multiprocess_mode: livemostrecent + - name: "dramstore_transport_fence_queue_size" + documentation: "Sampled aggregate queued Fence admission occupancy" + multiprocess_mode: livemostrecent + - name: "dramstore_transport_fence_queue_capacity" + documentation: "Aggregate reserved Fence admission capacity" + multiprocess_mode: livemostrecent + + # -- DramPool server observability ------------------------------------------ + - name: "drampool_metadata_entry_count" + documentation: "Current number of cached entries (blocks) in the pool, summed over all shards" + multiprocess_mode: livemostrecent + - name: "drampool_buffer_pool_usage_ratio" + documentation: "Used-slot ratio of the data pool per block size; exported as drampool_buffer_pool_usage_ratio{slot_size=}" + multiprocess_mode: livemostrecent + dynamic_labels: ["slot_size"] + - name: "drampool_flag_pool_usage_ratio" + documentation: "Used-slot ratio of the flag response-buffer pool" + multiprocess_mode: livemostrecent + - name: "drampool_queue_request_size" + documentation: "Current number of requests queued in requestQueue" + multiprocess_mode: livemostrecent + - name: "drampool_queue_completion_size" + documentation: "Current number of completion records queued in completionQueue" + multiprocess_mode: livemostrecent + - name: "drampool_queue_completion_inflight" + documentation: "Completion records currently in flight in the Poller pending window" + multiprocess_mode: livemostrecent + - name: "drampool_queue_request_capacity" + documentation: "Configured requestQueue depth (requests)" + multiprocess_mode: livemostrecent + - name: "drampool_queue_completion_capacity" + documentation: "Configured completionQueue depth (records)" + multiprocess_mode: livemostrecent + +# Histogram metrics configuration +histogram: + - name: "sglang_connector_load_pages" + documentation: "SGLang connector load: Successful storage batch size in pool pages; not deduplicated across TP/PP, pools or layers" + buckets: [1, 5, 10, 50, 100, 128, 256, 512] + - name: "sglang_connector_load_duration_ms" + documentation: "SGLang connector load: Storage submission-to-completion duration including failed attempts (ms); excludes buffer preparation and restoration" + buckets: [0.01, 0.1, 0.5, 1, 5, 10, 50, 100, 500, 1000, 5000, 10000] + - name: "sglang_connector_load_bandwidth_gbps" + documentation: "SGLang connector load: Successful call bytes divided by storage submission-to-completion time (GB/s, decimal bytes); not device-to-storage end-to-end bandwidth" + buckets: [0.1, 0.5, 1, 5, 10, 20, 50, 100] + - name: "sglang_connector_dump_pages" + documentation: "SGLang connector dump: Successful storage batch size in pool pages; not deduplicated across TP/PP, pools or layers" + buckets: [1, 5, 10, 50, 100, 128, 256, 512] + - name: "sglang_connector_dump_duration_ms" + documentation: "SGLang connector dump: Storage submission-to-completion duration including failed attempts (ms); excludes buffer preparation and restoration" + buckets: [0.01, 0.1, 0.5, 1, 5, 10, 50, 100, 500, 1000, 5000, 10000] + - name: "sglang_connector_dump_bandwidth_gbps" + documentation: "SGLang connector dump: Successful call bytes divided by storage submission-to-completion time (GB/s, decimal bytes); not device-to-storage end-to-end bandwidth" + buckets: [0.1, 0.5, 1, 5, 10, 20, 50, 100] + # BEGIN GENERATED KV histogram + # -- AsuStore observability ------------------------------------------------ + - name: "asu_lookup_duration_ms" + documentation: "AsuStore lookup call duration including key conversion, query wait and result processing (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 30000, 60000] + - name: "asu_load_submit_duration_ms" + documentation: "AsuStore Load submission duration including buffer conversion and failed submissions (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 30000, 60000] + - name: "asu_dump_submit_duration_ms" + documentation: "AsuStore Dump submission duration including buffer conversion and failed submissions (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 30000, 60000] + - name: "asu_wait_duration_ms" + documentation: "AsuStore Wait call duration including failed and timed-out waits (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 30000, 60000] + + - name: "kv_client_task_enqueue_duration_seconds" + documentation: "KV client API-to-enqueue duration" + buckets: [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0] + - name: "kv_client_task_queue_duration_seconds" + documentation: "KV client task queue duration" + buckets: [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0] + - name: "kv_client_task_process_duration_seconds" + documentation: "KV client task processing duration" + buckets: [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0] + - name: "kv_client_task_send_duration_seconds" + documentation: "KV client duration until all Send calls return" + buckets: [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0] + - name: "kv_client_task_e2e_duration_seconds" + documentation: "KV client task end-to-end duration" + buckets: [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0] + - name: "kv_transport_task_pre_send_duration_seconds" + documentation: "KV transport duration before Send" + buckets: [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0] + - name: "kv_transport_task_queue_duration_seconds" + documentation: "KV transport task queue duration" + buckets: [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0] + - name: "kv_transport_task_process_duration_seconds" + documentation: "KV transport task processing duration" + buckets: [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0] + - name: "kv_transport_task_send_duration_seconds" + documentation: "KV transport duration until Send returns" + buckets: [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0] + - name: "kv_transport_task_send_call_duration_seconds" + documentation: "KV transport provider Send call duration" + buckets: [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0] + - name: "kv_transport_task_completion_duration_seconds" + documentation: "KV transport completion duration after Send" + buckets: [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0] + - name: "kv_transport_node_task_send_duration_seconds" + documentation: "KV transport send duration by node" + buckets: [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0] + - name: "kv_transport_node_task_completion_duration_seconds" + documentation: "KV transport completion duration by node" + buckets: [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0] + - name: "kv_transport_task_e2e_duration_seconds" + documentation: "KV transport task end-to-end duration from submit to completion" + buckets: [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0] + # END GENERATED KV histogram + # -- End-to-end connector metrics ----------------------------------------- + - name: "save_duration" + documentation: "Time from UCM connector wait_for_save entry to async dump task completion (ms)" + buckets: [0, 50, 100, 150, 200, 250, 300, 350, 400, 550, 600, 750, 800, 850, 900, 950, 1000] + - name: "save_completion_wait_duration" + documentation: "Time spent blocked while confirming async UCM connector dump completion (ms)" + buckets: [0, 1, 2, 5, 10, 20, 50, 100, 150, 200, 250, 300, 350, 400, 550, 600, 750, 800, 850, 900, 950, 1000] + - name: "interval_lookup_hit_rates" + documentation: "Hit rates of ucm lookup requests" + buckets: [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0] + + # ========================================================================= + # Pipeline Store — per-stage metrics + # ========================================================================= + # These answer: "which tier in the Cache|Posix|Ds3fs stack is the bottleneck?" + # + # -- Cache stage (in-memory buffer on top of the stack) ------------------- + - name: "cache_lookup_duration_ms" + documentation: "Cache buffer lookup wall-clock time per `Lookup` / `LookupOnPrefix` call (ms)" + buckets: [0.01, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50, 100] + - name: "cache_lookup_backend_duration_ms" + documentation: "Backend lookup wall-clock time when descending due to no buffer or buffer miss (ms)" + buckets: [0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "cache_load_duration_ms" + documentation: "End-to-end Cache stage load task duration (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "cache_dump_duration_ms" + documentation: "End-to-end Cache stage dump task duration (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "cache_load_bandwidth_gbps" + documentation: "Cache stage effective load bandwidth (GB/s)" + buckets: [0.5, 1, 2, 4, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256] + - name: "cache_dump_bandwidth_gbps" + documentation: "Cache stage effective dump bandwidth (GB/s)" + buckets: [0.5, 1, 2, 4, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256] + - name: "cache_load_queue_wait_duration_ms" + documentation: "Time a Cache load task spent queued before dispatch worker pickup (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "cache_dump_queue_wait_duration_ms" + documentation: "Time a Cache dump task spent queued before dispatch worker pickup (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "cache_load_backend_submit_duration_ms" + documentation: "Cache load backend submit duration: buffer allocation plus synchronous backend load submission (ms)." + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "cache_shard_backend_wait_ms" + documentation: "Cache load per-shard time spent in WaitBackendTaskReady before H2D submit (ms). This is not a task-level duration." + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "cache_h2d_submit_ms" + documentation: "Cache load per-shard H2D async submit CPU cost after backend wait (ms). Submission only; NOT the actual transfer time (see cache_h2d_sync_ms)." + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "cache_h2d_sync_ms" + documentation: "Cache load residual H2D stream drain after the last shard submit (ms). Large => H2D copy is the bottleneck; ~0 with large cache_shard_backend_wait_ms => storage read is the bottleneck." + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "cache_dump_mkbuf_duration_ms" + documentation: "Cache dump mk_buf phase: buffer allocation/reuse + D2H async submit before stream sync (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "cache_dump_prereq_wait_ms" + documentation: "Cache dump time waiting for the prerequisite compute event (layer KV ready) to fire before D2H can start (ms). Large => dump is compute-gated, not copy-gated." + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "cache_d2h_duration_ms" + documentation: "Cache dump stream synchronize duration including prerequisite compute wait and D2H copy (ms). Use cache_dump_prereq_wait_ms to estimate the compute-gated portion." + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "cache_dump_backend_submit_duration_ms" + documentation: "Cache dump backend submit duration: synchronous time to pass buffers to the lower tier (ms). Does NOT include the lower tier's actual write time." + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "cache_dump_backend_wait_duration_ms" + documentation: "Cache dump time waiting for the lower tier to finish writing a dumped task (ms). Large => storage write is the bottleneck." + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000] + + # -- Posix stage (disk tier) --------------------------------------------- + - name: "posix_load_task_duration_ms" + documentation: "End-to-end Posix load task duration (ms): submit to last shard finished, task-level (compare with cache_load_duration_ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "posix_dump_task_duration_ms" + documentation: "End-to-end Posix dump task duration (ms): submit to last shard finished, task-level (compare with cache_dump_duration_ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + # Bucket layout: typical Posix throughput sits in 1-12 GB/s, so this range + # is sampled densely (0.5 GB/s steps for 1-4, 1 GB/s steps for 5-12) so + # that p50/p90/p99 bandwidth is meaningfully resolved. Sub-1 GB/s coverage + # is kept for detecting degraded paths; >12 GB/s is sparse for headroom. + - name: "posix_s2h_bandwidth_gbps" + documentation: "Posix stage read bandwidth per task (GB/s) = totalBytes / task_wallclock" + buckets: [0.05, 0.1, 0.2, 0.5, + 1, 1.5, 2, 2.5, 3, 3.5, 4, + 5, 6, 7, 8, 9, 10, 11, 12, + 14, 16, 20, 24, 32] + - name: "posix_h2s_bandwidth_gbps" + documentation: "Posix stage write bandwidth per task (GB/s) = totalBytes / task_wallclock" + buckets: [0.05, 0.1, 0.2, 0.5, + 1, 1.5, 2, 2.5, 3, 3.5, 4, + 5, 6, 7, 8, 9, 10, 11, 12, + 14, 16, 20, 24, 32] + - name: "posix_load_queue_wait_duration_ms" + documentation: "Time a Posix load task spent queued before first worker pickup (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "posix_dump_queue_wait_duration_ms" + documentation: "Time a Posix dump task spent queued before first worker pickup (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + + # -- Mooncake stage ------------------------------------------------------- + - name: "mooncake_load_duration_ms" + documentation: "End-to-end Mooncake load task duration (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "mooncake_dump_duration_ms" + documentation: "End-to-end Mooncake dump task duration (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "mooncake_load_bandwidth_gbps" + documentation: "Mooncake stage effective load bandwidth (GB/s)" + buckets: [0.5, 1, 2, 4, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256] + - name: "mooncake_dump_bandwidth_gbps" + documentation: "Mooncake stage effective dump bandwidth (GB/s)" + buckets: [0.5, 1, 2, 4, 8, 12, 16, 24, 32, 48, 64, 96, 128, 192, 256] + - name: "mooncake_load_queue_wait_duration_ms" + documentation: "Time a Mooncake load task spent queued before dispatch worker pickup (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "mooncake_dump_queue_wait_duration_ms" + documentation: "Time a Mooncake dump task spent queued before dispatch worker pickup (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "mooncake_get_duration_ms" + documentation: "Mooncake batch get duration on the load path (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "mooncake_exists_duration_ms" + documentation: "Mooncake batch exists check duration on the dump path (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "mooncake_put_duration_ms" + documentation: "Mooncake batch put duration on the dump path (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000] + - name: "mooncake_load_backend_submit_duration_ms" + documentation: "Mooncake load backend submit duration after Mooncake miss (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "mooncake_backend_load_wait_duration_ms" + documentation: "Mooncake load time waiting for the backend to finish a missed shard (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "mooncake_h2d_duration_ms" + documentation: "Mooncake load H2D stream drain duration (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "mooncake_dump_prereq_wait_ms" + documentation: "Mooncake dump time waiting for prerequisite compute event before put (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "mooncake_d2h_duration_ms" + documentation: "Mooncake dump D2H stream drain duration for backend archive (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "mooncake_dump_backend_submit_duration_ms" + documentation: "Mooncake dump backend submit duration after D2H archive copy (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "mooncake_dump_backend_wait_duration_ms" + documentation: "Mooncake dump time waiting for backend archive completion (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000] + + # vLLM Connector public interfaces + - name: "connector_get_block_size_duration_ms" + documentation: "Wall-clock duration of UCMConnector.get_block_size invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_get_kv_connector_stats_duration_ms" + documentation: "Wall-clock duration of UCMConnector.get_kv_connector_stats invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_get_num_new_matched_tokens_duration_ms" + documentation: "Wall-clock duration of UCMConnector.get_num_new_matched_tokens invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_update_state_after_alloc_duration_ms" + documentation: "Wall-clock duration of UCMConnector.update_state_after_alloc invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_register_kv_caches_duration_ms" + documentation: "Wall-clock duration of UCMConnector.register_kv_caches invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_build_connector_meta_duration_ms" + documentation: "Wall-clock duration of UCMConnector.build_connector_meta invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_bind_connector_metadata_duration_ms" + documentation: "Wall-clock duration of UCMConnector.bind_connector_metadata invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_handle_preemptions_duration_ms" + documentation: "Wall-clock duration of UCMConnector.handle_preemptions invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_has_connector_metadata_duration_ms" + documentation: "Wall-clock duration of UCMConnector.has_connector_metadata invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_start_load_kv_duration_ms" + documentation: "Wall-clock duration of UCMConnector.start_load_kv invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_wait_for_layer_load_duration_ms" + documentation: "Wall-clock duration of UCMConnector.wait_for_layer_load invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_save_kv_layer_duration_ms" + documentation: "Wall-clock duration of UCMConnector.save_kv_layer invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_wait_for_save_duration_ms" + documentation: "Wall-clock duration of UCMConnector.wait_for_save invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_request_finished_all_groups_duration_ms" + documentation: "Wall-clock duration of UCMConnector.request_finished_all_groups invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_request_finished_duration_ms" + documentation: "Wall-clock duration of UCMConnector.request_finished invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_get_finished_duration_ms" + documentation: "Wall-clock duration of UCMConnector.get_finished invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_build_connector_worker_meta_duration_ms" + documentation: "Wall-clock duration of UCMConnector.build_connector_worker_meta invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_update_connector_output_duration_ms" + documentation: "Wall-clock duration of UCMConnector.update_connector_output invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_clear_connector_metadata_duration_ms" + documentation: "Wall-clock duration of UCMConnector.clear_connector_metadata invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "connector_get_block_ids_with_load_errors_duration_ms" + documentation: "Wall-clock duration of UCMConnector.get_block_ids_with_load_errors invoked by vLLM (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + + # ========================================================================= + # Layerwise Connector — overlap / pipelining health + # ========================================================================= + - name: "layerwise_layer_load_duration_ms" + documentation: "Layerwise per-layer wall-clock time from layer load start to wait_for_layer_load return (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "layerwise_batch_load_duration_sum_ms" + documentation: "Sum of per-layer load durations within one Layerwise batch (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + + # -- DramStore client observability ----------------------------------------- + - name: "dramstore_lookup_duration_ms" + documentation: "End-to-end DramStore lookup duration (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "dramstore_lookup_task_queue_duration_ms" + documentation: "DramStore lookup task queue duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "dramstore_lookup_task_to_request_duration_ms" + documentation: "DramStore lookup duration from task admission through request construction (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "dramstore_lookup_request_duration_ms" + documentation: "DramStore lookup request duration (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "dramstore_lookup_request_queue_duration_ms" + documentation: "DramStore lookup request NodeScheduler queue duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "dramstore_lookup_request_pending_duration_ms" + documentation: "DramStore lookup request NodeActor pending duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "dramstore_lookup_request_setup_duration_ms" + documentation: "DramStore lookup request successful setup duration including reply-slot allocation and encoding (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "dramstore_lookup_request_transport_queue_duration_ms" + documentation: "DramStore lookup request TransportExecutor queue duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "dramstore_lookup_request_transport_send_duration_ms" + documentation: "DramStore lookup request TCP send duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000] + - name: "dramstore_lookup_request_remote_duration_ms" + documentation: "DramStore lookup request remote processing and reply duration (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "dramstore_dump_duration_ms" + documentation: "End-to-end DramStore dump duration (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "dramstore_dump_task_queue_duration_ms" + documentation: "DramStore dump task queue duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "dramstore_dump_task_to_request_duration_ms" + documentation: "DramStore dump duration from task admission through request construction (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "dramstore_dump_request_duration_ms" + documentation: "DramStore dump request duration (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "dramstore_dump_request_queue_duration_ms" + documentation: "DramStore dump request NodeScheduler queue duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "dramstore_dump_request_pending_duration_ms" + documentation: "DramStore dump request NodeActor pending duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "dramstore_dump_request_setup_duration_ms" + documentation: "DramStore dump request successful setup duration including reply-slot allocation and encoding (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "dramstore_dump_request_transport_queue_duration_ms" + documentation: "DramStore dump request TransportExecutor queue duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "dramstore_dump_request_transport_send_duration_ms" + documentation: "DramStore dump request TCP send duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000] + - name: "dramstore_dump_request_remote_duration_ms" + documentation: "DramStore dump request remote processing and reply duration (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "dramstore_load_duration_ms" + documentation: "End-to-end DramStore load duration (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "dramstore_load_task_queue_duration_ms" + documentation: "DramStore load task queue duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "dramstore_load_task_to_request_duration_ms" + documentation: "DramStore load duration from task admission through request construction (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "dramstore_load_request_duration_ms" + documentation: "DramStore load request duration (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "dramstore_load_request_queue_duration_ms" + documentation: "DramStore load request NodeScheduler queue duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "dramstore_load_request_pending_duration_ms" + documentation: "DramStore load request NodeActor pending duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "dramstore_load_request_setup_duration_ms" + documentation: "DramStore load request successful setup duration including reply-slot allocation and encoding (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + - name: "dramstore_load_request_transport_queue_duration_ms" + documentation: "DramStore load request TransportExecutor queue duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "dramstore_load_request_transport_send_duration_ms" + documentation: "DramStore load request TCP send duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000] + - name: "dramstore_load_request_remote_duration_ms" + documentation: "DramStore load request remote processing and reply duration (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "dramstore_dump_prerequisite_duration_ms" + documentation: "DUMP prerequisite event wait before task admission (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500] + + # -- DramPool server observability ------------------------------------------ + - name: "drampool_dump_prepare_duration_ms" + documentation: "DUMP local preparation duration per batch: StoreBegin per entry, buffer allocation, eviction retries, and transfer submission (ms, successful path only; split into dump_metadata and dump_submit)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "drampool_dump_metadata_duration_ms" + documentation: "DUMP metadata-processing segment per batch: StoreBegin per entry, buffer allocation, eviction retries, and transfer-item building, excluding the ExecuteAsync submission (ms, successful path only)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "drampool_dump_submit_duration_ms" + documentation: "DUMP data-transfer ExecuteAsync submission duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "drampool_load_prepare_duration_ms" + documentation: "LOAD local preparation duration per batch: LoadBegin per entry, length validation, and transfer submission (ms, successful path only; split into load_metadata and load_submit)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "drampool_load_metadata_duration_ms" + documentation: "LOAD metadata-processing segment per batch: LoadBegin per entry, length validation, and transfer-item building, excluding the ExecuteAsync submission (ms, successful path only)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "drampool_load_submit_duration_ms" + documentation: "LOAD data-transfer ExecuteAsync submission duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "drampool_lookup_scan_duration_ms" + documentation: "Per-batch LOOKUP metadata scan duration: per-entry existence checks plus result filling (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "drampool_dump_transfer_duration_ms" + documentation: "DUMP data transfer duration from ExecuteAsync return to terminal-state observation, including HiXL transfer, polling delay, and GetStatus (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "drampool_load_transfer_duration_ms" + documentation: "LOAD data transfer duration from ExecuteAsync return to terminal-state observation, including HiXL transfer, polling delay, and GetStatus (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "drampool_get_status_duration_ms" + documentation: "GetStatus() execution duration when the Poller queries data or response transfers (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "drampool_response_submit_duration_ms" + documentation: "Response write-back ExecuteAsync submission duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "drampool_response_rtt_ms" + documentation: "Response round-trip duration from response-transfer ExecuteAsync return to write-back completion in client memory, including polling delay and GetStatus but excluding the submission itself (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "drampool_metadata_storeend_duration_ms" + documentation: "Per-entry StoreEnd settlement duration after DUMP transfer completion (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "drampool_metadata_loadend_duration_ms" + documentation: "Per-entry LoadEnd reference-release duration (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "drampool_metadata_evict_gc_duration_ms" + documentation: "Duration of one background GC eviction sweep across all shards (ms)" + buckets: [1, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000, 10000] + - name: "drampool_queue_request_enqueue_wait_ms" + documentation: "Per-request wait from preparation to successful requestQueue TryPush, including full-queue retries (ms)" + buckets: [0.01, 0.05, 0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 500] + - name: "drampool_queue_request_residence_ms" + documentation: "Per-request residence in requestQueue from the successful TryPush to the TaskWorker dequeue, excluding the TryPush wait (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "drampool_dump_batch_total_duration_ms" + documentation: "End-to-end DUMP batch duration across the full server-side request lifecycle: from requestQueue push to response-transfer terminal state, or the record leaving the Poller on a permanent response failure (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "drampool_load_batch_total_duration_ms" + documentation: "End-to-end LOAD batch duration across the full server-side request lifecycle: from requestQueue push to response-transfer terminal state, or the record leaving the Poller on a permanent response failure (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + - name: "drampool_lookup_batch_total_duration_ms" + documentation: "End-to-end LOOKUP batch duration across the full server-side request lifecycle: from requestQueue push to response-transfer terminal state, or the record leaving the Poller on a permanent response failure (ms)" + buckets: [0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 5000] + # Direct linker: per-worker counters and asynchronous operation timing. + - name: sglang_linker_lookup_duration_ms + documentation: 'SGLang linker: Owner adapter lookup duration including failed attempts (ms)' + buckets: + - 0.01 + - 0.1 + - 0.5 + - 1 + - 5 + - 10 + - 50 + - 100 + - 500 + - 1000 + - 5000 + - 10000 + - name: sglang_linker_load_queue_duration_ms + documentation: 'SGLang linker load: Enqueue to worker dequeue time; excludes scheduler pending-load + residence (ms)' + buckets: + - 0.01 + - 0.1 + - 0.5 + - 1 + - 5 + - 10 + - 50 + - 100 + - 500 + - 1000 + - 5000 + - 10000 + - name: sglang_linker_load_duration_ms + documentation: 'SGLang linker load: Enqueue to worker completion time including queueing, preparation + and failures; excludes ordered-publication delay and downstream background persistence (ms)' + buckets: + - 0.01 + - 0.1 + - 0.5 + - 1 + - 5 + - 10 + - 50 + - 100 + - 500 + - 1000 + - 5000 + - 10000 + - name: sglang_linker_offload_queue_duration_ms + documentation: 'SGLang linker offload: Enqueue to worker dequeue time; excludes scheduler pending-load + residence (ms)' + buckets: + - 0.01 + - 0.1 + - 0.5 + - 1 + - 5 + - 10 + - 50 + - 100 + - 500 + - 1000 + - 5000 + - 10000 + - name: sglang_linker_offload_duration_ms + documentation: 'SGLang linker offload: Enqueue to worker completion time including queueing, preparation + and failures; excludes ordered-publication delay and downstream background persistence (ms)' + buckets: + - 0.01 + - 0.1 + - 0.5 + - 1 + - 5 + - 10 + - 50 + - 100 + - 500 + - 1000 + - 5000 + - 10000 diff --git a/examples/metrics/sglang-dashboard.json b/examples/metrics/sglang-dashboard.json new file mode 100644 index 000000000..3142fbb63 --- /dev/null +++ b/examples/metrics/sglang-dashboard.json @@ -0,0 +1,1517 @@ +{ + "id": null, + "uid": "sglang-dashboard", + "title": "SGLang + UCM / Overview", + "schemaVersion": 39, + "version": 1, + "editable": true, + "timezone": "browser", + "refresh": "30s", + "tags": [ + "sglang", + "ucm" + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "label": "datasource", + "type": "datasource", + "query": "prometheus", + "refresh": 1, + "options": [], + "hide": 0 + }, + { + "name": "sglang_prefix", + "label": "SGLang prefix", + "type": "textbox", + "query": "sglang:", + "current": { + "text": "sglang:", + "value": "sglang:" + }, + "hide": 2 + }, + { + "name": "ucm_prefix", + "label": "UCM prefix", + "type": "textbox", + "query": "ucm:", + "current": { + "text": "ucm:", + "value": "ucm:" + }, + "hide": 2 + }, + { + "name": "model_name", + "label": "model_name", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${sglang_prefix}.*|${ucm_prefix}.*\",model_name!=\"\"}, model_name)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${sglang_prefix}.*|${ucm_prefix}.*\",model_name!=\"\"}, model_name)", + "refresh": 2, + "sort": 1, + "multi": false, + "includeAll": false, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "job", + "label": "job", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${sglang_prefix}.*|${ucm_prefix}.*\",model_name=~\"$model_name\"}, job)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${sglang_prefix}.*|${ucm_prefix}.*\",model_name=~\"$model_name\"}, job)", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + }, + { + "name": "instance", + "label": "instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${sglang_prefix}.*|${ucm_prefix}.*\",model_name=~\"$model_name\",job=~\"$job\"}, instance)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${sglang_prefix}.*|${ucm_prefix}.*\",model_name=~\"$model_name\",job=~\"$job\"}, instance)", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + }, + { + "name": "worker_id", + "label": "worker_id", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${ucm_prefix}.*\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}, worker_id)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${ucm_prefix}.*\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}, worker_id)", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + } + ] + }, + "panels": [ + { + "type": "text", + "title": "About this dashboard", + "options": { + "mode": "markdown", + "content": "Serving performance and UCM summary. Native serving panels use job / instance / model_name; worker_id filters UCM metrics. Connector throughput counts successful bytes, while request duration includes failed attempts. Storage health=1 means the breaker is enabled, not that every I/O succeeded. Open HiCache for cache scheduling, or Connector & Store for physical I/O and backend diagnostics. Missing series remain No data." + }, + "id": 1, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 4 + } + }, + { + "type": "row", + "title": "SGLang / serving performance", + "collapsed": false, + "panels": [], + "id": 2, + "gridPos": { + "x": 0, + "y": 4, + "w": 24, + "h": 1 + } + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 5, + "w": 12, + "h": 8 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(${sglang_prefix}e2e_request_latency_seconds_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P99", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(${sglang_prefix}e2e_request_latency_seconds_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P90", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(${sglang_prefix}e2e_request_latency_seconds_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P50", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(rate(${sglang_prefix}e2e_request_latency_seconds_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) / sum(rate(${sglang_prefix}e2e_request_latency_seconds_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval]))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "End-to-End Request Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 5, + "w": 12, + "h": 8 + }, + "id": 4, + "maxDataPoints": 30, + "options": { + "calculate": false, + "calculation": { + "yBuckets": { + "scale": { + "type": "linear" + } + } + }, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "secs" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(increase(${sglang_prefix}e2e_request_latency_seconds_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) by (le)\r\n", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "End-to-End Request Latency(s) Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 13, + "w": 12, + "h": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le) (rate(${sglang_prefix}time_to_first_token_seconds_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P99", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le) (rate(${sglang_prefix}time_to_first_token_seconds_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P90", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le) (rate(${sglang_prefix}time_to_first_token_seconds_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])))\r\n", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "P50", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(rate(${sglang_prefix}time_to_first_token_seconds_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval])) / sum(rate(${sglang_prefix}time_to_first_token_seconds_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval]))", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Avg", + "range": true, + "refId": "D", + "useBackend": false + } + ], + "title": "Time-To-First-Token Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + }, + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 13, + "w": 12, + "h": 8 + }, + "id": 6, + "maxDataPoints": 30, + "options": { + "calculate": false, + "calculation": { + "xBuckets": { + "value": "" + }, + "yBuckets": { + "mode": "size", + "scale": { + "type": "linear" + }, + "value": "" + } + }, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": true, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(le) (increase(${sglang_prefix}time_to_first_token_seconds_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}[$__rate_interval]))", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Time-To-First-Token Seconds Heatmap", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 21, + "w": 12, + "h": 8 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "${sglang_prefix}num_running_reqs{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Num Running Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 21, + "w": 12, + "h": 8 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "${sglang_prefix}gen_throughput{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}", + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Token Generation Throughput (Tokens / S)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 29, + "w": 12, + "h": 8 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "${sglang_prefix}cache_hit_rate{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Cache Hit Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 29, + "w": 12, + "h": 8 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.6.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "${sglang_prefix}num_queue_reqs{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Number Queued Requests", + "type": "timeseries" + }, + { + "type": "row", + "title": "UCM / transfer and storage health", + "collapsed": false, + "panels": [], + "id": 11, + "gridPos": { + "x": 0, + "y": 37, + "w": 24, + "h": 1 + } + }, + { + "title": "Successful transfer throughput", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Successful bytes per second over the selected rate window, including idle time. Values are local to each worker.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_bytes_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load bytes {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_bytes_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump bytes {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 12, + "gridPos": { + "x": 0, + "y": 38, + "w": 12, + "h": 8 + } + }, + { + "title": "Storage submission and completion failures", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Failed calls per second, separated by load/dump and submit/wait. Missing series means no sample has been emitted; it is not replaced with zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_submit_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load submit errors {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_wait_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load wait errors {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_submit_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump submit errors {{instance}} {{worker_id}}", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_wait_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump wait errors {{instance}} {{worker_id}}", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 13, + "gridPos": { + "x": 12, + "y": 38, + "w": 12, + "h": 8 + } + }, + { + "title": "Mean storage call duration", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Mean time from storage submission to confirmed completion, including failed attempts. Excludes earlier queueing, buffer preparation and restoration.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load duration ms {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump duration ms {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 14, + "gridPos": { + "x": 0, + "y": 46, + "w": 12, + "h": 8 + } + }, + { + "title": "Posix storage health", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Effective Posix health breaker state, where 1 is enabled and 0 is fused\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}posix_store_health{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Fused", + "color": "red" + }, + "1": { + "text": "Enabled", + "color": "green" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 15, + "gridPos": { + "x": 12, + "y": 46, + "w": 12, + "h": 8 + } + }, + { + "title": "Posix garbage collection", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Posix garbage collection state, where 1 is running and 0 is idle\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}posix_gc_running{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 16, + "gridPos": { + "x": 0, + "y": 54, + "w": 12, + "h": 8 + } + }, + { + "title": "Posix I/O error rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Posix read, write, or AIO completion failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}posix_io_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 17, + "gridPos": { + "x": 12, + "y": 54, + "w": 12, + "h": 8 + } + } + ], + "links": [ + { + "type": "link", + "title": "SGLang + UCM / HiCache", + "url": "/d/sglang-ucm-hicache", + "includeVars": true, + "keepTime": true, + "targetBlank": false + }, + { + "type": "link", + "title": "SGLang + UCM / Connector & Store", + "url": "/d/sglang-ucm-store", + "includeVars": true, + "keepTime": true, + "targetBlank": false + } + ] +} diff --git a/examples/metrics/sglang-hicache-dashboard.json b/examples/metrics/sglang-hicache-dashboard.json new file mode 100644 index 000000000..0cccbbde0 --- /dev/null +++ b/examples/metrics/sglang-hicache-dashboard.json @@ -0,0 +1,1327 @@ +{ + "id": null, + "uid": "sglang-ucm-hicache", + "title": "SGLang + UCM / HiCache", + "schemaVersion": 39, + "version": 1, + "editable": true, + "timezone": "browser", + "refresh": "30s", + "tags": [ + "sglang", + "ucm" + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "label": "datasource", + "type": "datasource", + "query": "prometheus", + "refresh": 1, + "options": [], + "hide": 0 + }, + { + "name": "sglang_prefix", + "label": "SGLang prefix", + "type": "textbox", + "query": "sglang:", + "current": { + "text": "sglang:", + "value": "sglang:" + }, + "hide": 2 + }, + { + "name": "model_name", + "label": "model_name", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${sglang_prefix}.*\",model_name!=\"\"}, model_name)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${sglang_prefix}.*\",model_name!=\"\"}, model_name)", + "refresh": 2, + "sort": 1, + "multi": false, + "includeAll": false, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "job", + "label": "job", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${sglang_prefix}.*\",model_name=~\"$model_name\"}, job)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${sglang_prefix}.*\",model_name=~\"$model_name\"}, job)", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + }, + { + "name": "instance", + "label": "instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${sglang_prefix}.*\",model_name=~\"$model_name\",job=~\"$job\"}, instance)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${sglang_prefix}.*\",model_name=~\"$model_name\",job=~\"$job\"}, instance)", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + }, + { + "name": "tp_rank", + "label": "tp_rank", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${sglang_prefix}.*\",job=~\"$job\",instance=~\"$instance\"}, tp_rank)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${sglang_prefix}.*\",job=~\"$job\",instance=~\"$instance\"}, tp_rank)", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + }, + { + "name": "pp_rank", + "label": "pp_rank", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${sglang_prefix}.*\",job=~\"$job\",instance=~\"$instance\"}, pp_rank)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${sglang_prefix}.*\",job=~\"$job\",instance=~\"$instance\"}, pp_rank)", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + }, + { + "name": "dp_rank", + "label": "dp_rank", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${sglang_prefix}.*\",job=~\"$job\",instance=~\"$instance\"}, dp_rank)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${sglang_prefix}.*\",job=~\"$job\",instance=~\"$instance\"}, dp_rank)", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + } + ] + }, + "panels": [ + { + "type": "text", + "title": "About this dashboard", + "options": { + "mode": "markdown", + "content": "HiCache scheduling and cache behavior only: L2 device/host transfers, eviction, and L3 prefetch/backup. Use job, instance and TP/PP/DP rank filters. Token counts retain rank and pool dimensions and are not globally deduplicated. Physical UCM storage-call metrics are on the Connector & Store dashboard. No Linker-specific metrics are included. Missing series remain No data." + }, + "id": 1, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 4 + } + }, + { + "type": "row", + "title": "L2 host cache / load, backup and eviction", + "collapsed": false, + "panels": [], + "id": 2, + "gridPos": { + "x": 0, + "y": 4, + "w": 24, + "h": 1 + } + }, + { + "title": "Host cache eviction duration", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "End-to-end time of a device eviction pass in seconds; under --hicache-write-policy write_back this includes the blocking D->H backup (see sglang:hicache_backup_duration_seconds for the copy alone).\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}eviction_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval])) / sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}eviction_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.5, sum by (le, job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}eviction_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval])))", + "legendFormat": "__auto", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}eviction_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval])))", + "legendFormat": "__auto", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}eviction_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval])))", + "legendFormat": "__auto", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 3, + "gridPos": { + "x": 0, + "y": 5, + "w": 12, + "h": 8 + } + }, + { + "title": "Evicted host-cache tokens", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The number of device KV token slots freed by eviction, regardless of whether the data was backed up to host (see sglang:hicache_backup_tokens_total) or destroyed (see sglang:hicache_dropped_tokens_total).\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}evicted_tokens_total{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: tokens/s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 4, + "gridPos": { + "x": 12, + "y": 5, + "w": 12, + "h": 8 + } + }, + { + "title": "Host-to-device load duration", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "GPU-stream span of a merged host-to-device load-back copy loop, including gaps between per-layer I/O submissions but excluding pre-transfer queue and fence waits.\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}load_back_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval])) / sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}load_back_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.5, sum by (le, job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}load_back_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval])))", + "legendFormat": "__auto", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}load_back_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval])))", + "legendFormat": "__auto", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}load_back_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval])))", + "legendFormat": "__auto", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 5, + "gridPos": { + "x": 0, + "y": 13, + "w": 12, + "h": 8 + } + }, + { + "title": "Host-to-device loaded tokens", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The number of tokens loaded back from local host DRAM (L2) to GPU, by host pool (kv, swa, mamba, ...). Every TP rank reports the same logical count under its own rank labels; read one rank rather than summing ranks.\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}load_back_tokens_total{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: tokens/s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 6, + "gridPos": { + "x": 12, + "y": 13, + "w": 12, + "h": 8 + } + }, + { + "title": "Device-to-host backup duration", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Time taken to back up KV cache from GPU to local host DRAM (L2) in seconds, per merged write op. Covers all D->H backups regardless of --hicache-write-policy. Distinct from the host-to-storage (L3) sglang:backuped_tokens_total.\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}hicache_backup_duration_seconds_sum{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval])) / sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}hicache_backup_duration_seconds_count{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.5, sum by (le, job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}hicache_backup_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval])))", + "legendFormat": "__auto", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}hicache_backup_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval])))", + "legendFormat": "__auto", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}hicache_backup_duration_seconds_bucket{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval])))", + "legendFormat": "__auto", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 7, + "gridPos": { + "x": 0, + "y": 21, + "w": 12, + "h": 8 + } + }, + { + "title": "Device-to-host backup throughput", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Bytes backed up from GPU to local host DRAM (L2), all pools combined, including draft/sidecar transfers that the token counter excludes. Each rank reports its own KV shard; sum ranks for the physical total. A window's byte delta divided by the same window's hicache_backup_duration_seconds sum is the bytes-weighted active bandwidth; the plain rate is wall-clock payload traffic that includes idle time.\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}hicache_backup_bytes_total{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 8, + "gridPos": { + "x": 12, + "y": 21, + "w": 12, + "h": 8 + } + }, + { + "title": "Host-to-device load throughput", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Bytes loaded back from local host DRAM (L2) to GPU, all pools combined, including draft/sidecar transfers that the token counter excludes. Each rank reports its own KV shard; sum ranks for the physical total. A window's byte delta divided by the same window's load_back_duration_seconds sum is the bytes-weighted active bandwidth; the plain rate is wall-clock payload traffic that includes idle time.\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}load_back_bytes_total{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 9, + "gridPos": { + "x": 0, + "y": 29, + "w": 12, + "h": 8 + } + }, + { + "title": "Device-to-host backed-up tokens", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The number of tokens backed up from GPU to local host DRAM (L2), by host pool (kv, swa, mamba, ...). Covers all D->H backups regardless of --hicache-write-policy. Distinct from the host-to-storage (L3) sglang:backuped_tokens_total. Every TP rank reports the same logical count under its own rank labels; read one rank rather than summing ranks.\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}hicache_backup_tokens_total{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: tokens/s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 10, + "gridPos": { + "x": 12, + "y": 29, + "w": 12, + "h": 8 + } + }, + { + "title": "Dropped host-cache tokens", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "The number of logical device KV tokens destroyed without a host backup. The pool label is always kv; reason is host_pressure for write-back failure, or write_through_unbacked_eviction when eager write-through did not complete before eviction.\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}hicache_dropped_tokens_total{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: tokens/s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 11, + "gridPos": { + "x": 0, + "y": 37, + "w": 12, + "h": 8 + } + }, + { + "type": "row", + "title": "L3 storage / prefetch and backup", + "collapsed": false, + "panels": [], + "id": 12, + "gridPos": { + "x": 0, + "y": 45, + "w": 24, + "h": 1 + } + }, + { + "title": "L3 prefetched tokens", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Prompt tokens successfully transferred from L3 storage into host memory, before admission-time reuse or discard.\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}prefetched_tokens_total{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: tokens/s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 13, + "gridPos": { + "x": 0, + "y": 46, + "w": 12, + "h": 8 + } + }, + { + "title": "L3 backed-up tokens", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of backuped tokens.\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}backuped_tokens_total{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: tokens/s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 14, + "gridPos": { + "x": 12, + "y": 46, + "w": 12, + "h": 8 + } + }, + { + "title": "L3 prefetch hits", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Storage-hit tokens returned by an L3 query before host allocation, transfer, or cache insertion.\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}storage_prefetch_hit_tokens_total{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: tokens/s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 15, + "gridPos": { + "x": 0, + "y": 54, + "w": 12, + "h": 8 + } + }, + { + "title": "L3 unfulfilled prefetch", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Attempt-level storage-hit tokens that did not become a usable prefetch result, by diagnostic reason. This is not a final prefill cache-miss counter. anchor_lost means the original device prefix was no longer present when the prefetch needed it; aux_window_trim means a staged aux trailing window could not be trimmed to the shorter splice, so the loaded span was released; device_overlap means the live device prefix diverged from the request's view at splice time, so the span beyond it was released; cache_admission_shortfall means an L3-loaded L2 span was no longer reusable when cache-mode admission ran; deadline means the storage prefetch stop policy ended the read before it completed.\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}storage_prefetch_unfulfilled_tokens_total{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: tokens/s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 16, + "gridPos": { + "x": 12, + "y": 54, + "w": 12, + "h": 8 + } + }, + { + "title": "L3 deferred prefetch", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Storage-prefetch token-attempts deferred for later reuse, by transient capacity reason.\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}storage_prefetch_deferred_tokens_total{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: tokens/s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 17, + "gridPos": { + "x": 0, + "y": 62, + "w": 12, + "h": 8 + } + }, + { + "title": "Dropped L3 backup tokens", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Buffer-mode backup tokens that never reached L3 because admission rejected them or the source became stale before the D2H staging launched; the write path lagging device churn is the common cause, and the span rewrites only on a later recompute.\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}hicache_backup_dropped_tokens_total{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: tokens/s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 18, + "gridPos": { + "x": 12, + "y": 62, + "w": 12, + "h": 8 + } + }, + { + "title": "Prefetch auxiliary allocation failures", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Prefetch tokens abandoned because an aux pool (e.g. the SWA trailing window) could not allocate host staging — the whole storage fetch is forfeited, not just the aux part.\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, tp_rank, pp_rank, dp_rank, pool, reason, storage_backend, cache_type) (rate(${sglang_prefix}hicache_prefetch_aux_alloc_failed_tokens_total{job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: tokens/s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 19, + "gridPos": { + "x": 0, + "y": 70, + "w": 12, + "h": 8 + } + }, + { + "type": "row", + "title": "HiCache / cumulative counters and observations", + "collapsed": true, + "panels": [ + { + "title": "HiCache cumulative counter", + "type": "table", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Cumulative samples per rank; logical token counts must not be summed across TP ranks.", + "targets": [ + { + "expr": "{__name__=~\"${sglang_prefix}(prefetched_tokens_total|backuped_tokens_total|storage_prefetch_hit_tokens_total|storage_prefetch_unfulfilled_tokens_total|storage_prefetch_deferred_tokens_total|hicache_backup_dropped_tokens_total|hicache_prefetch_aux_alloc_failed_tokens_total|evicted_tokens_total|load_back_tokens_total|hicache_backup_bytes_total|load_back_bytes_total|hicache_backup_tokens_total|hicache_dropped_tokens_total)\",job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": true, + "range": false, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "id": 21, + "gridPos": { + "x": 0, + "y": 79, + "w": 24, + "h": 8 + } + }, + { + "title": "HiCache cumulative histogram", + "type": "table", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Cumulative samples per rank; logical token counts must not be summed across TP ranks.", + "targets": [ + { + "expr": "{__name__=~\"${sglang_prefix}(eviction_duration_seconds_count|load_back_duration_seconds_count|hicache_backup_duration_seconds_count)\",job=~\"$job\",instance=~\"$instance\",tp_rank=~\"$tp_rank\",pp_rank=~\"$pp_rank\",dp_rank=~\"$dp_rank\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": true, + "range": false, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "id": 22, + "gridPos": { + "x": 0, + "y": 87, + "w": 24, + "h": 8 + } + } + ], + "id": 20, + "gridPos": { + "x": 0, + "y": 78, + "w": 24, + "h": 1 + } + } + ], + "links": [ + { + "type": "link", + "title": "SGLang + UCM / Overview", + "url": "/d/sglang-dashboard", + "includeVars": true, + "keepTime": true, + "targetBlank": false + }, + { + "type": "link", + "title": "SGLang + UCM / Connector & Store", + "url": "/d/sglang-ucm-store", + "includeVars": true, + "keepTime": true, + "targetBlank": false + } + ] +} diff --git a/examples/metrics/sglang-linker-dashboard.json b/examples/metrics/sglang-linker-dashboard.json new file mode 100644 index 000000000..1243d50ae --- /dev/null +++ b/examples/metrics/sglang-linker-dashboard.json @@ -0,0 +1,2025 @@ +{ + "id": null, + "uid": "sglang-ucm-linker", + "title": "SGLang + UCM / Direct Linker", + "schemaVersion": 39, + "version": 1, + "editable": true, + "timezone": "browser", + "refresh": "30s", + "tags": [ + "sglang", + "ucm", + "linker" + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "label": "datasource", + "type": "datasource", + "query": "prometheus", + "refresh": 1, + "options": [], + "hide": 0 + }, + { + "name": "ucm_prefix", + "label": "UCM prefix", + "type": "textbox", + "query": "ucm:", + "current": { + "text": "ucm:", + "value": "ucm:" + }, + "hide": 2 + }, + { + "name": "model_name", + "label": "model_name", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${ucm_prefix}.*\",model_name!=\"\"}, model_name)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${ucm_prefix}.*\",model_name!=\"\"}, model_name)", + "refresh": 2, + "sort": 1, + "multi": false, + "includeAll": false, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "job", + "label": "job", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${ucm_prefix}.*\",model_name=~\"$model_name\"}, job)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${ucm_prefix}.*\",model_name=~\"$model_name\"}, job)", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + }, + { + "name": "instance", + "label": "instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${ucm_prefix}.*\",model_name=~\"$model_name\",job=~\"$job\"}, instance)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${ucm_prefix}.*\",model_name=~\"$model_name\",job=~\"$job\"}, instance)", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + }, + { + "name": "worker_id", + "label": "worker_id", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${ucm_prefix}.*\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}, worker_id)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${ucm_prefix}.*\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}, worker_id)", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + } + ] + }, + "panels": [ + { + "type": "text", + "title": "Metric boundaries", + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 5 + }, + "options": { + "mode": "markdown", + "content": "Direct L1 linker metrics, grouped by worker. **Lookup coverage is the maximum locally restorable page boundary on successful owner queries**, before cross-rank intersection; repeated/resumed queries count again. Non-owner acknowledgements are excluded from hits and I/O. Load operations are batches; offload operations are owner nodes. Queue time starts at worker-queue insertion (not scheduler pending-load admission). Operation latency ends at worker completion, before ordered result publication. Physical I/O is measured per pool/layer shard, from submission to Wait confirmation; pages are not deduplicated across layers or ranks. **Cache completion does not imply downstream Posix persistence**. Check failures are also included in connector completion errors; do not add those two series. Queue estimates can briefly lag because export is periodic. GB/s means decimal bytes/s. Missing series remain No data; no zero-fill or synthetic latency samples." + }, + "id": 1 + }, + { + "type": "row", + "title": "Owner lookup / local restorable coverage", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 5, + "w": 24, + "h": 1 + }, + "id": 2 + }, + { + "title": "Lookup attempts and failures", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_lookup_requests_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "requests {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_lookup_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "errors {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 3, + "gridPos": { + "x": 0, + "y": 6, + "w": 12, + "h": 8 + } + }, + { + "title": "Local restorable page coverage", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Successful owner queries only; not final cache token hit rate. Zero queries give no valid ratio.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_lookup_hit_pages_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_lookup_query_pages_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "local coverage {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 4, + "gridPos": { + "x": 12, + "y": 6, + "w": 12, + "h": 8 + } + }, + { + "title": "Requested and locally restorable pages", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Full query spans, including resumed prefixes and repeated lookups.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_lookup_query_pages_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "query {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_lookup_hit_pages_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "hit {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "pps", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 5, + "gridPos": { + "x": 0, + "y": 14, + "w": 12, + "h": 8 + } + }, + { + "title": "Owner lookup duration", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_lookup_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_lookup_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "mean {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_lookup_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p95 {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 6, + "gridPos": { + "x": 12, + "y": 14, + "w": 12, + "h": 8 + } + }, + { + "title": "Resume, fallback and non-owner lookup", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Fallback counts per-group probes; resumed counts owner API attempts.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_lookup_resumed_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "resumed {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_lookup_fallback_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "fallback {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_lookup_skipped_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "skipped {{instance}} {{worker_id}}", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 7, + "gridPos": { + "x": 0, + "y": 22, + "w": 12, + "h": 8 + } + }, + { + "title": "Load requests and cancellations", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_load_requests_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "accepted requests {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_load_cancelled_requests_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "cancelled before batching {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 8, + "gridPos": { + "x": 12, + "y": 22, + "w": 12, + "h": 8 + } + }, + { + "type": "row", + "title": "Linker workers / queueing, completion and errors", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 30, + "w": 24, + "h": 1 + }, + "id": 9 + }, + { + "title": "Load batches / activity", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_load_operations_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "operations {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_load_started_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "started {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_load_completed_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "completed {{instance}} {{worker_id}}", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_load_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "errors {{instance}} {{worker_id}}", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 10, + "gridPos": { + "x": 0, + "y": 31, + "w": 12, + "h": 8 + } + }, + { + "title": "Load batches / outstanding work", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Counter differences per worker, not cross-thread Gauge snapshots. Periodic export can transiently skew the estimate.", + "targets": [ + { + "expr": "clamp_min(sum by (job, instance, model_name, worker_id) (${ucm_prefix}sglang_linker_load_operations_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}) - sum by (job, instance, model_name, worker_id) (${ucm_prefix}sglang_linker_load_started_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}), 0)", + "legendFormat": "queued {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "clamp_min(sum by (job, instance, model_name, worker_id) (${ucm_prefix}sglang_linker_load_started_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}) - sum by (job, instance, model_name, worker_id) (${ucm_prefix}sglang_linker_load_completed_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}), 0)", + "legendFormat": "active {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 11, + "gridPos": { + "x": 12, + "y": 31, + "w": 12, + "h": 8 + } + }, + { + "title": "Load batches / queue time", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_load_queue_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_load_queue_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "mean {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_load_queue_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p95 {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 12, + "gridPos": { + "x": 0, + "y": 39, + "w": 12, + "h": 8 + } + }, + { + "title": "Load batches / enqueue-to-completion time", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_load_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_load_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "mean {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_load_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p95 {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 13, + "gridPos": { + "x": 12, + "y": 39, + "w": 12, + "h": 8 + } + }, + { + "title": "Offload nodes / activity", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_offload_operations_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "operations {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_offload_started_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "started {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_offload_completed_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "completed {{instance}} {{worker_id}}", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_offload_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "errors {{instance}} {{worker_id}}", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 14, + "gridPos": { + "x": 0, + "y": 47, + "w": 12, + "h": 8 + } + }, + { + "title": "Offload nodes / outstanding work", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Counter differences per worker, not cross-thread Gauge snapshots. Periodic export can transiently skew the estimate.", + "targets": [ + { + "expr": "clamp_min(sum by (job, instance, model_name, worker_id) (${ucm_prefix}sglang_linker_offload_operations_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}) - sum by (job, instance, model_name, worker_id) (${ucm_prefix}sglang_linker_offload_started_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}), 0)", + "legendFormat": "queued {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "clamp_min(sum by (job, instance, model_name, worker_id) (${ucm_prefix}sglang_linker_offload_started_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}) - sum by (job, instance, model_name, worker_id) (${ucm_prefix}sglang_linker_offload_completed_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}), 0)", + "legendFormat": "active {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 15, + "gridPos": { + "x": 12, + "y": 47, + "w": 12, + "h": 8 + } + }, + { + "title": "Offload nodes / queue time", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_offload_queue_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_offload_queue_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "mean {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_offload_queue_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p95 {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 16, + "gridPos": { + "x": 0, + "y": 55, + "w": 12, + "h": 8 + } + }, + { + "title": "Offload nodes / enqueue-to-completion time", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_offload_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_offload_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "mean {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_offload_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p95 {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 17, + "gridPos": { + "x": 12, + "y": 55, + "w": 12, + "h": 8 + } + }, + { + "title": "Layers ready and non-owner offloads", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_load_ready_layers_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "logical layers ready {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_offload_skipped_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "non-owner offload acknowledgements {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 18, + "gridPos": { + "x": 0, + "y": 63, + "w": 12, + "h": 8 + } + }, + { + "title": "Store Check failures", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Also included in connector wait errors once reaped. Successful cleanup does not erase a Check failure.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_load_check_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_linker_dump_check_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 19, + "gridPos": { + "x": 12, + "y": 63, + "w": 12, + "h": 8 + } + }, + { + "type": "row", + "title": "Physical Store transfers / pool and layer shards", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 71, + "w": 24, + "h": 1 + }, + "id": 20 + }, + { + "title": "Physical storage call rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Actual storage calls, recorded at final completion, including failed submissions. Not inference requests.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_requests_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_requests_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 21, + "gridPos": { + "x": 0, + "y": 72, + "w": 12, + "h": 8 + } + }, + { + "title": "Successful transferred bytes per second", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_bytes_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_bytes_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 22, + "gridPos": { + "x": 12, + "y": 72, + "w": 12, + "h": 8 + } + }, + { + "title": "Submission and completion failures", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_submit_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load submit {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_wait_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load wait {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_submit_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump submit {{instance}} {{worker_id}}", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_wait_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump wait {{instance}} {{worker_id}}", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 23, + "gridPos": { + "x": 0, + "y": 80, + "w": 12, + "h": 8 + } + }, + { + "title": "Mean physical pages per successful call", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Page count repeats for each physical layer/pool call; do not use it as logical hit pages.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_pages_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_pages_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_pages_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_pages_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 24, + "gridPos": { + "x": 12, + "y": 80, + "w": 12, + "h": 8 + } + }, + { + "title": "Load / submit-to-confirmation duration", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "mean {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p95 {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 25, + "gridPos": { + "x": 0, + "y": 88, + "w": 12, + "h": 8 + } + }, + { + "title": "Load / successful call bandwidth (GB/s)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_bandwidth_gbps_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_bandwidth_gbps_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "mean {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p95 {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 26, + "gridPos": { + "x": 12, + "y": 88, + "w": 12, + "h": 8 + } + }, + { + "title": "Dump / submit-to-confirmation duration", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "mean {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p95 {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 27, + "gridPos": { + "x": 0, + "y": 96, + "w": 12, + "h": 8 + } + }, + { + "title": "Dump / successful call bandwidth (GB/s)", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_bandwidth_gbps_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_bandwidth_gbps_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "mean {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p95 {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "none", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 28, + "gridPos": { + "x": 12, + "y": 96, + "w": 12, + "h": 8 + } + }, + { + "type": "row", + "title": "Cumulative observations / useful immediately after startup", + "collapsed": false, + "panels": [], + "gridPos": { + "x": 0, + "y": 104, + "w": 24, + "h": 1 + }, + "id": 29 + }, + { + "title": "Linker cumulative counters", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Counters reset with worker restart. Retain each worker to avoid TP multiplication of logical pages.", + "targets": [ + { + "expr": "{__name__=~\"${ucm_prefix}sglang_linker_.*_total\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "{{__name__}} {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 30, + "gridPos": { + "x": 0, + "y": 105, + "w": 12, + "h": 8 + } + }, + { + "title": "Physical I/O cumulative counters", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "targets": [ + { + "expr": "{__name__=~\"${ucm_prefix}sglang_connector_.*_total\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "{{__name__}} {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 31, + "gridPos": { + "x": 12, + "y": 105, + "w": 12, + "h": 8 + } + } + ], + "links": [ + { + "type": "link", + "title": "SGLang + UCM / Overview", + "url": "/d/sglang-dashboard", + "includeVars": true, + "keepTime": true, + "targetBlank": false + }, + { + "type": "link", + "title": "SGLang + UCM / Connector & Store", + "url": "/d/sglang-ucm-store", + "includeVars": true, + "keepTime": true, + "targetBlank": false + }, + { + "type": "link", + "title": "SGLang + UCM / HiCache", + "url": "/d/sglang-ucm-hicache", + "includeVars": true, + "keepTime": true, + "targetBlank": false + } + ] +} diff --git a/examples/metrics/sglang-ucm-store-dashboard.json b/examples/metrics/sglang-ucm-store-dashboard.json new file mode 100644 index 000000000..e416d229f --- /dev/null +++ b/examples/metrics/sglang-ucm-store-dashboard.json @@ -0,0 +1,12639 @@ +{ + "id": null, + "uid": "sglang-ucm-store", + "title": "SGLang + UCM / Connector & Store", + "schemaVersion": 39, + "version": 1, + "editable": true, + "timezone": "browser", + "refresh": "30s", + "tags": [ + "sglang", + "ucm" + ], + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "DS_PROMETHEUS", + "label": "datasource", + "type": "datasource", + "query": "prometheus", + "refresh": 1, + "options": [], + "hide": 0 + }, + { + "name": "ucm_prefix", + "label": "UCM prefix", + "type": "textbox", + "query": "ucm:", + "current": { + "text": "ucm:", + "value": "ucm:" + }, + "hide": 2 + }, + { + "name": "model_name", + "label": "model_name", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${ucm_prefix}.*\",model_name!=\"\"}, model_name)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${ucm_prefix}.*\",model_name!=\"\"}, model_name)", + "refresh": 2, + "sort": 1, + "multi": false, + "includeAll": false, + "current": {}, + "options": [], + "hide": 0 + }, + { + "name": "job", + "label": "job", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${ucm_prefix}.*\",model_name=~\"$model_name\"}, job)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${ucm_prefix}.*\",model_name=~\"$model_name\"}, job)", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + }, + { + "name": "instance", + "label": "instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${ucm_prefix}.*\",model_name=~\"$model_name\",job=~\"$job\"}, instance)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${ucm_prefix}.*\",model_name=~\"$model_name\",job=~\"$job\"}, instance)", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + }, + { + "name": "worker_id", + "label": "worker_id", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values({__name__=~\"${ucm_prefix}.*\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}, worker_id)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values({__name__=~\"${ucm_prefix}.*\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\"}, worker_id)", + "refresh": 2, + "sort": 1, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "options": [] + }, + { + "name": "perWorker", + "label": "Store aggregation", + "type": "custom", + "query": "Per worker : job\\, instance\\, model_name\\, worker_id,Per instance : job\\, instance\\, model_name", + "current": { + "text": "Per worker", + "value": "job, instance, model_name, worker_id" + }, + "options": [ + { + "text": "Per worker", + "value": "job, instance, model_name, worker_id", + "selected": true + }, + { + "text": "Per instance", + "value": "job, instance, model_name", + "selected": false + } + ] + } + ] + }, + "panels": [ + { + "type": "text", + "title": "About this dashboard", + "options": { + "mode": "markdown", + "content": "Physical UCM storage calls and backend diagnostics for HiCache and direct Linker. Call count and duration include failures; pages, bytes and bandwidth describe successful calls only. HiCache records per physical pool call; Linker records per pool/layer shard. Neither is inference request count or deduplicated logical page count. Each worker reports its own transferred data. HiCache currently selects Posix; Linker supports its configured pipeline, including Cache|Posix. Cache task completion does not imply downstream Posix persistence. Store task byte/duration metrics can include failed tasks. Multiple Store groups share per-worker Counter/Histogram series; existing same-name Gauges do not identify groups and must not be interpreted as total capacity or all-group health. Shared-resource observations across workers must not be summed blindly. Missing series remain No data." + }, + "id": 1, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 4 + } + }, + { + "type": "row", + "title": "Connector / activity and failures", + "collapsed": false, + "panels": [], + "id": 2, + "gridPos": { + "x": 0, + "y": 4, + "w": 24, + "h": 1 + } + }, + { + "title": "Storage call rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Physical storage calls attempted, including failures. One call can cover multiple pages; this is not inference request rate.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_requests_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load requests {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_requests_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump requests {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 3, + "gridPos": { + "x": 0, + "y": 5, + "w": 12, + "h": 8 + } + }, + { + "title": "Successful transfer throughput", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Successful bytes per second over the selected rate window, including idle time. Values are local to each worker.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_bytes_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load bytes {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_bytes_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump bytes {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 4, + "gridPos": { + "x": 12, + "y": 5, + "w": 12, + "h": 8 + } + }, + { + "title": "Storage submission and completion failures", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Failed calls per second, separated by load/dump and submit/wait. Missing series means no sample has been emitted; it is not replaced with zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_submit_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load submit errors {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_wait_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load wait errors {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_submit_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump submit errors {{instance}} {{worker_id}}", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_wait_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump wait errors {{instance}} {{worker_id}}", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 5, + "gridPos": { + "x": 0, + "y": 13, + "w": 12, + "h": 8 + } + }, + { + "title": "Mean storage call duration", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Mean time from storage submission to confirmed completion, including failed attempts. Excludes earlier queueing, buffer preparation and restoration.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load duration ms {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump duration ms {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 6, + "gridPos": { + "x": 12, + "y": 13, + "w": 12, + "h": 8 + } + }, + { + "type": "row", + "title": "Connector / transfer size, latency and bandwidth distributions", + "collapsed": true, + "panels": [ + { + "title": "Load call duration", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "SGLang connector load: Storage submission-to-completion duration including failed attempts (ms); excludes buffer preparation and restoration\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "Mean {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.5, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P50 {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P95 {{instance}} {{worker_id}}", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P99 {{instance}} {{worker_id}}", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 8, + "gridPos": { + "x": 0, + "y": 22, + "w": 12, + "h": 8 + } + }, + { + "title": "Dump call duration", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "SGLang connector dump: Storage submission-to-completion duration including failed attempts (ms); excludes buffer preparation and restoration\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "Mean {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.5, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P50 {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P95 {{instance}} {{worker_id}}", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P99 {{instance}} {{worker_id}}", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 9, + "gridPos": { + "x": 12, + "y": 22, + "w": 12, + "h": 8 + } + }, + { + "title": "Load successful pages per call", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "SGLang connector load: Successful storage batch size in pool pages; not deduplicated across TP/PP, pools or layers\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_pages_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_pages_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "Mean {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.5, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_pages_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P50 {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_pages_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P95 {{instance}} {{worker_id}}", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_pages_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P99 {{instance}} {{worker_id}}", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 10, + "gridPos": { + "x": 0, + "y": 30, + "w": 12, + "h": 8 + } + }, + { + "title": "Dump successful pages per call", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "SGLang connector dump: Successful storage batch size in pool pages; not deduplicated across TP/PP, pools or layers\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_pages_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_pages_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "Mean {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.5, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_pages_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P50 {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_pages_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P95 {{instance}} {{worker_id}}", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_pages_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P99 {{instance}} {{worker_id}}", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 11, + "gridPos": { + "x": 12, + "y": 30, + "w": 12, + "h": 8 + } + }, + { + "title": "Load successful per-call bandwidth", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "SGLang connector load: Successful call bytes divided by storage submission-to-completion time (GB/s, decimal bytes); not device-to-storage end-to-end bandwidth\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_bandwidth_gbps_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_bandwidth_gbps_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "Mean {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.5, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P50 {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P95 {{instance}} {{worker_id}}", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_load_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P99 {{instance}} {{worker_id}}", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: GB/s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 12, + "gridPos": { + "x": 0, + "y": 38, + "w": 12, + "h": 8 + } + }, + { + "title": "Dump successful per-call bandwidth", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "SGLang connector dump: Successful call bytes divided by storage submission-to-completion time (GB/s, decimal bytes); not device-to-storage end-to-end bandwidth\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_bandwidth_gbps_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_bandwidth_gbps_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "Mean {{instance}} {{worker_id}}", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.5, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P50 {{instance}} {{worker_id}}", + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P95 {{instance}} {{worker_id}}", + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, job, instance, model_name, worker_id) (rate(${ucm_prefix}sglang_connector_dump_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "P99 {{instance}} {{worker_id}}", + "refId": "D", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "suffix: GB/s", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 13, + "gridPos": { + "x": 12, + "y": 38, + "w": 12, + "h": 8 + } + } + ], + "id": 7, + "gridPos": { + "x": 0, + "y": 21, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Connector / cumulative counters and observations", + "collapsed": true, + "panels": [ + { + "title": "Connector cumulative counters", + "type": "table", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Cumulative values since worker start. Histogram count is the number of observations, not a success count.", + "targets": [ + { + "expr": "{__name__=~\"${ucm_prefix}(sglang_connector_load_requests_total|sglang_connector_load_bytes_total|sglang_connector_load_submit_errors_total|sglang_connector_load_wait_errors_total|sglang_connector_dump_requests_total|sglang_connector_dump_bytes_total|sglang_connector_dump_submit_errors_total|sglang_connector_dump_wait_errors_total)\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": true, + "range": false, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "id": 15, + "gridPos": { + "x": 0, + "y": 23, + "w": 24, + "h": 8 + } + }, + { + "title": "Connector cumulative observations", + "type": "table", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Cumulative values since worker start. Histogram count is the number of observations, not a success count.", + "targets": [ + { + "expr": "{__name__=~\"${ucm_prefix}(sglang_connector_load_pages_count|sglang_connector_load_duration_ms_count|sglang_connector_load_bandwidth_gbps_count|sglang_connector_dump_pages_count|sglang_connector_dump_duration_ms_count|sglang_connector_dump_bandwidth_gbps_count)\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": true, + "range": false, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "id": 16, + "gridPos": { + "x": 0, + "y": 31, + "w": 24, + "h": 8 + } + } + ], + "id": 14, + "gridPos": { + "x": 0, + "y": 22, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Posix / Health and resources", + "collapsed": true, + "panels": [ + { + "title": "posix_healthy_count_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of successful Posix health probes\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}posix_healthy_count_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 18, + "gridPos": { + "x": 0, + "y": 24, + "w": 12, + "h": 8 + } + }, + { + "title": "posix_store_used_bytes", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Estimated logical Posix Store usage in bytes from GC sampling\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}posix_store_used_bytes{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 19, + "gridPos": { + "x": 12, + "y": 24, + "w": 12, + "h": 8 + } + }, + { + "title": "posix_store_capacity_bytes", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Configured logical Posix Store capacity in bytes\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}posix_store_capacity_bytes{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 20, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + } + }, + { + "title": "posix_store_usage_ratio", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Estimated logical Posix Store usage ratio\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}posix_store_usage_ratio{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 21, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + } + }, + { + "title": "posix_store_health", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Effective Posix health breaker state, where 1 is enabled and 0 is fused\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}posix_store_health{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Fused", + "color": "red" + }, + "1": { + "text": "Enabled", + "color": "green" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 22, + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + } + }, + { + "title": "posix_gc_running", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Posix garbage collection state, where 1 is running and 0 is idle\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}posix_gc_running{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 23, + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 8 + } + } + ], + "id": 17, + "gridPos": { + "x": 0, + "y": 23, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Posix / Throughput and latency", + "collapsed": true, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:posix_load_task_duration_ms` 记录 Posix Load 任务端到端耗时;本面板使用 Histogram 的 `_count` 增速表示每秒完成的 Load 任务数。代码位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch`,以及 `ucm/store/posix/cc/io_engine_psync.h` / `IoEnginePsync::Dispatch` / waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: tasks/s", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 25, + "w": 12, + "h": 8 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_load_task_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "load {{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Posix Load Task Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:posix_dump_task_duration_ms` 记录 Posix Dump 任务端到端耗时;本面板使用 Histogram 的 `_count` 增速表示每秒完成的 Dump 任务数。代码位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch`,以及 `ucm/store/posix/cc/io_engine_psync.h` / `IoEnginePsync::Dispatch` / waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: tasks/s", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 25, + "w": 12, + "h": 8 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_dump_task_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "dump {{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Posix Dump Task Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:posix_load_task_duration_ms` 表示 Posix Load 任务从提交到最后一个 shard 完成的端到端耗时,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch`,以及 `ucm/store/posix/cc/io_engine_psync.h` / `IoEnginePsync::Dispatch` / waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 33, + "w": 12, + "h": 8 + }, + "id": 27, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_load_task_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_load_task_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_load_task_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_load_task_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}posix_load_task_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Posix Load Task Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:posix_dump_task_duration_ms` 表示 Posix Dump 任务从提交到最后一个 shard 完成的端到端耗时,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch`,以及 `ucm/store/posix/cc/io_engine_psync.h` / `IoEnginePsync::Dispatch` / waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 12, + "y": 33, + "w": 12, + "h": 8 + }, + "id": 28, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_dump_task_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_dump_task_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_dump_task_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_dump_task_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}posix_dump_task_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Posix Dump Task Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标与代码位置:\n- `ucm:posix_load_queue_wait_duration_ms`:Load 任务从提交到开始执行 IO 的排队时间。AIO 位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch` / IO 分发前;PSYNC 位置:`ucm/store/posix/cc/trans_queue.cc` / `TransQueue::LoadWorker` / 首个 IO unit 开始时。\n- `ucm:posix_load_task_duration_ms`:Load 任务端到端耗时。位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch`,以及 `ucm/store/posix/cc/io_engine_psync.h` / `IoEnginePsync::Dispatch` / waiter 完成回调。\n`task after queue` 为总耗时减去排队时间后的估算值。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 45, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 41, + "w": 12, + "h": 8 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_load_queue_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}posix_load_queue_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))", + "legendFormat": "queue wait {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "clamp_min(\n (\n sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_load_task_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n /\n sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_load_task_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n )\n -\n (\n sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_load_queue_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n /\n sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_load_queue_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n ),\n 0\n)", + "legendFormat": "task after queue {{worker_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Posix Load Avg Breakdown", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标与代码位置:\n- `ucm:posix_dump_queue_wait_duration_ms`:Dump 任务从提交到开始执行 IO 的排队时间。AIO 位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch` / IO 分发前;PSYNC 位置:`ucm/store/posix/cc/trans_queue.cc` / `TransQueue::DumpWorker` / 首个 IO unit 开始时。\n- `ucm:posix_dump_task_duration_ms`:Dump 任务端到端耗时。位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch`,以及 `ucm/store/posix/cc/io_engine_psync.h` / `IoEnginePsync::Dispatch` / waiter 完成回调。\n`task after queue` 为总耗时减去排队时间后的估算值。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 45, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 41, + "w": 12, + "h": 8 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_dump_queue_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}posix_dump_queue_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))", + "legendFormat": "queue wait {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "clamp_min(\n (\n sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_dump_task_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n /\n sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_dump_task_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n )\n -\n (\n sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_dump_queue_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n /\n sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_dump_queue_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n ),\n 0\n)", + "legendFormat": "task after queue {{worker_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Posix Dump Avg Breakdown", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:posix_s2h_bytes_total` 表示 Posix Load 从存储读取到 Host 的累计字节数;本面板用字节增速计算每个 worker 的聚合 GB/s,包含空闲间隔。代码位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch`,以及 `ucm/store/posix/cc/io_engine_psync.h` / `IoEnginePsync::Dispatch` / Load waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: GB/s", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 49, + "w": 12, + "h": 8 + }, + "id": 31, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_s2h_bytes_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / 1e9", + "legendFormat": " {{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Posix S2H Bandwidth (aggregated)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:posix_h2s_bytes_total` 表示 Posix Dump 从 Host 写入存储的累计字节数;本面板用字节增速计算每个 worker 的聚合 GB/s,包含空闲间隔。代码位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch`,以及 `ucm/store/posix/cc/io_engine_psync.h` / `IoEnginePsync::Dispatch` / Dump waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: GB/s", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 49, + "w": 12, + "h": 8 + }, + "id": 32, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_h2s_bytes_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / 1e9", + "legendFormat": " {{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Posix H2S Bandwidth (aggregated)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:posix_load_queue_wait_duration_ms` 表示 Posix Load 任务从提交到开始执行 IO 的排队时间,本面板展示 p50、p90、p99 和平均值。AIO 代码位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch` / IO 分发前;PSYNC 代码位置:`ucm/store/posix/cc/trans_queue.cc` / `TransQueue::LoadWorker` / 首个 IO unit 开始时。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 57, + "w": 12, + "h": 8 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_load_queue_wait_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_load_queue_wait_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_load_queue_wait_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_load_queue_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}posix_load_queue_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Posix Load Queue Wait Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:posix_dump_queue_wait_duration_ms` 表示 Posix Dump 任务从提交到开始执行 IO 的排队时间,本面板展示 p50、p90、p99 和平均值。AIO 代码位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch` / IO 分发前;PSYNC 代码位置:`ucm/store/posix/cc/trans_queue.cc` / `TransQueue::DumpWorker` / 首个 IO unit 开始时。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 12, + "y": 57, + "w": 12, + "h": 8 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_dump_queue_wait_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_dump_queue_wait_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_dump_queue_wait_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_dump_queue_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}posix_dump_queue_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Posix Dump Queue Wait Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:posix_s2h_bandwidth_gbps` 表示单个 Posix Load 任务的字节数除以端到端耗时所得带宽,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch`,以及 `ucm/store/posix/cc/io_engine_psync.h` / `IoEnginePsync::Dispatch` / Load waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: GB/s", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 65, + "w": 12, + "h": 8 + }, + "id": 35, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_s2h_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_s2h_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_s2h_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_s2h_bandwidth_gbps_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}posix_s2h_bandwidth_gbps_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Posix S2H Bandwidth (per task)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:posix_h2s_bandwidth_gbps` 表示单个 Posix Dump 任务的字节数除以端到端耗时所得带宽,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch`,以及 `ucm/store/posix/cc/io_engine_psync.h` / `IoEnginePsync::Dispatch` / Dump waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: GB/s", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 12, + "y": 65, + "w": 12, + "h": 8 + }, + "id": 36, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_h2s_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_h2s_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}posix_h2s_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}posix_h2s_bandwidth_gbps_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}posix_h2s_bandwidth_gbps_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Posix H2S Bandwidth (per task)", + "type": "timeseries" + }, + { + "type": "heatmap", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 37, + "title": "Posix / Load Task Duration", + "description": "指标:`ucm:posix_load_task_duration_ms` 表示 Posix Load 任务端到端耗时;热力图使用 bucket 增速显示各耗时区间的任务速率。代码位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch`,以及 `ucm/store/posix/cc/io_engine_psync.h` / `IoEnginePsync::Dispatch` / Load waiter 完成回调。", + "gridPos": { + "x": 0, + "y": 73, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (le) (rate(${ucm_prefix}posix_load_task_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "options": { + "calculate": false, + "yAxis": { + "axisPlacement": "left", + "unit": "ms", + "decimals": 0, + "reverse": false + }, + "rowsFrame": { + "layout": "auto", + "value": "Request rate" + }, + "color": { + "mode": "scheme", + "scheme": "Spectral", + "fill": "dark-orange", + "exponent": 0.5, + "steps": 64, + "reverse": false + }, + "cellGap": 1, + "cellValues": { + "unit": "suffix: req/s" + }, + "filterValues": { + "le": 1e-09 + }, + "tooltip": { + "show": true, + "yHistogram": true, + "mode": "single" + }, + "legend": { + "show": true + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "noValue": "No data" + }, + "overrides": [] + }, + "pluginVersion": "10.4.0" + }, + { + "type": "heatmap", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 38, + "title": "Posix / Dump Task Duration", + "description": "指标:`ucm:posix_dump_task_duration_ms` 表示 Posix Dump 任务端到端耗时;热力图使用 bucket 增速显示各耗时区间的任务速率。代码位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch`,以及 `ucm/store/posix/cc/io_engine_psync.h` / `IoEnginePsync::Dispatch` / Dump waiter 完成回调。", + "gridPos": { + "x": 12, + "y": 73, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (le) (rate(${ucm_prefix}posix_dump_task_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "options": { + "calculate": false, + "yAxis": { + "axisPlacement": "left", + "unit": "ms", + "decimals": 0, + "reverse": false + }, + "rowsFrame": { + "layout": "auto", + "value": "Request rate" + }, + "color": { + "mode": "scheme", + "scheme": "Spectral", + "fill": "dark-orange", + "exponent": 0.5, + "steps": 64, + "reverse": false + }, + "cellGap": 1, + "cellValues": { + "unit": "suffix: req/s" + }, + "filterValues": { + "le": 1e-09 + }, + "tooltip": { + "show": true, + "yHistogram": true, + "mode": "single" + }, + "legend": { + "show": true + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "noValue": "No data" + }, + "overrides": [] + }, + "pluginVersion": "10.4.0" + }, + { + "type": "heatmap", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 39, + "title": "Posix / S2H Bandwidth (per task)", + "description": "指标:`ucm:posix_s2h_bandwidth_gbps` 表示单个 Posix Load 任务的任务级带宽;热力图使用 bucket 增速显示各带宽区间的任务速率。代码位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch`,以及 `ucm/store/posix/cc/io_engine_psync.h` / `IoEnginePsync::Dispatch` / Load waiter 完成回调。", + "gridPos": { + "x": 0, + "y": 81, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (le) (rate(${ucm_prefix}posix_s2h_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "options": { + "calculate": false, + "yAxis": { + "axisPlacement": "left", + "unit": "Bps", + "decimals": 0, + "reverse": false + }, + "rowsFrame": { + "layout": "auto", + "value": "Request rate" + }, + "color": { + "mode": "scheme", + "scheme": "Spectral", + "fill": "dark-orange", + "exponent": 0.5, + "steps": 64, + "reverse": false + }, + "cellGap": 1, + "cellValues": { + "unit": "suffix: req/s" + }, + "filterValues": { + "le": 1e-09 + }, + "tooltip": { + "show": true, + "yHistogram": true, + "mode": "single" + }, + "legend": { + "show": true + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "noValue": "No data", + "unit": "suffix: GB/s" + }, + "overrides": [] + }, + "pluginVersion": "10.4.0" + }, + { + "type": "heatmap", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 40, + "title": "Posix / H2S Bandwidth (per task)", + "description": "指标:`ucm:posix_h2s_bandwidth_gbps` 表示单个 Posix Dump 任务的任务级带宽;热力图使用 bucket 增速显示各带宽区间的任务速率。代码位置:`ucm/store/posix/cc/io_engine_aio.h` / `IoEngineAio::Dispatch`,以及 `ucm/store/posix/cc/io_engine_psync.h` / `IoEnginePsync::Dispatch` / Dump waiter 完成回调。", + "gridPos": { + "x": 12, + "y": 81, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (le) (rate(${ucm_prefix}posix_h2s_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "options": { + "calculate": false, + "yAxis": { + "axisPlacement": "left", + "unit": "Bps", + "decimals": 0, + "reverse": false + }, + "rowsFrame": { + "layout": "auto", + "value": "Request rate" + }, + "color": { + "mode": "scheme", + "scheme": "Spectral", + "fill": "dark-orange", + "exponent": 0.5, + "steps": 64, + "reverse": false + }, + "cellGap": 1, + "cellValues": { + "unit": "suffix: req/s" + }, + "filterValues": { + "le": 1e-09 + }, + "tooltip": { + "show": true, + "yHistogram": true, + "mode": "single" + }, + "legend": { + "show": true + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "noValue": "No data", + "unit": "suffix: GB/s" + }, + "overrides": [] + }, + "pluginVersion": "10.4.0" + }, + { + "title": "posix_lookup_query_blocks_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total blocks submitted to Posix lookup\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}posix_lookup_query_blocks_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 41, + "gridPos": { + "x": 0, + "y": 89, + "w": 12, + "h": 8 + } + }, + { + "title": "posix_lookup_hit_blocks_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Blocks found by Posix lookup\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}posix_lookup_hit_blocks_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 42, + "gridPos": { + "x": 12, + "y": 89, + "w": 12, + "h": 8 + } + } + ], + "id": 24, + "gridPos": { + "x": 0, + "y": 24, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Posix / Errors and recovery", + "collapsed": true, + "panels": [ + { + "title": "posix_unhealthy_count_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of failed Posix health probes\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}posix_unhealthy_count_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 44, + "gridPos": { + "x": 0, + "y": 26, + "w": 12, + "h": 8 + } + }, + { + "title": "posix_passive_failures_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of failed Posix IO tasks observed by passive health detection\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}posix_passive_failures_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 45, + "gridPos": { + "x": 12, + "y": 26, + "w": 12, + "h": 8 + } + }, + { + "title": "posix_aio_timeout_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Posix AIO task or submit timeouts\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}posix_aio_timeout_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 46, + "gridPos": { + "x": 0, + "y": 34, + "w": 12, + "h": 8 + } + }, + { + "title": "posix_io_timeout_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Posix synchronous worker task timeouts\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}posix_io_timeout_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 47, + "gridPos": { + "x": 12, + "y": 34, + "w": 12, + "h": 8 + } + }, + { + "title": "posix_open_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Posix open failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}posix_open_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 48, + "gridPos": { + "x": 0, + "y": 42, + "w": 12, + "h": 8 + } + }, + { + "title": "posix_io_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Posix read, write, or AIO completion failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}posix_io_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 49, + "gridPos": { + "x": 12, + "y": 42, + "w": 12, + "h": 8 + } + } + ], + "id": 43, + "gridPos": { + "x": 0, + "y": 25, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Posix / Cumulative counters and observations", + "collapsed": true, + "panels": [ + { + "title": "Posix / cumulative counter samples", + "type": "table", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Cumulative values since worker start. Histogram count is the number of observations, not a success count.", + "targets": [ + { + "expr": "{__name__=~\"${ucm_prefix}(posix_s2h_bytes_total|posix_h2s_bytes_total|posix_lookup_query_blocks_total|posix_lookup_hit_blocks_total|posix_healthy_count_total|posix_unhealthy_count_total|posix_passive_failures_total|posix_aio_timeout_total|posix_io_timeout_total|posix_open_errors_total|posix_io_errors_total)\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": true, + "range": false, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "id": 51, + "gridPos": { + "x": 0, + "y": 27, + "w": 24, + "h": 8 + } + }, + { + "title": "Posix / cumulative histogram samples", + "type": "table", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Cumulative values since worker start. Histogram count is the number of observations, not a success count.", + "targets": [ + { + "expr": "{__name__=~\"${ucm_prefix}(posix_load_task_duration_ms_count|posix_dump_task_duration_ms_count|posix_s2h_bandwidth_gbps_count|posix_h2s_bandwidth_gbps_count|posix_load_queue_wait_duration_ms_count|posix_dump_queue_wait_duration_ms_count)\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": true, + "range": false, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "id": 52, + "gridPos": { + "x": 0, + "y": 35, + "w": 24, + "h": 8 + } + } + ], + "id": 50, + "gridPos": { + "x": 0, + "y": 26, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Cache / Throughput and latency", + "collapsed": true, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_load_duration_ms` 记录 Cache Load 任务端到端耗时;本面板使用 Histogram 的 `_count` 增速表示每秒完成的 Load 任务数。代码位置:`ucm/store/cache/cc/trans_manager.h` / `TransManager::Dispatch` / waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: tasks/s", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 28, + "w": 12, + "h": 8 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "load {{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Load Task Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_dump_duration_ms` 记录 Cache Dump 任务端到端耗时;本面板使用 Histogram 的 `_count` 增速表示每秒完成的 Dump 任务数。代码位置:`ucm/store/cache/cc/trans_manager.h` / `TransManager::Dispatch` / waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: tasks/s", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 28, + "w": 12, + "h": 8 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "dump {{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Dump Task Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_load_duration_ms` 表示 Cache Load 任务从提交到完成的端到端耗时,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/cache/cc/trans_manager.h` / `TransManager::Dispatch` / waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 36, + "w": 12, + "h": 8 + }, + "id": 56, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_load_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_load_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_load_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Load Total Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_dump_duration_ms` 表示 Cache Dump 任务从提交到完成的端到端耗时,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/cache/cc/trans_manager.h` / `TransManager::Dispatch` / waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 12, + "y": 36, + "w": 12, + "h": 8 + }, + "id": 57, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Dump Total Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "- `queue wait`:指标 `ucm:cache_load_queue_wait_duration_ms`;Load 任务从提交到调度线程取出的等待时间。代码位置:`ucm/store/cache/cc/load_queue.cc` / `LoadQueue::DispatchOneTask` / 开始处理任务时。\n- `backend submit`:指标 `ucm:cache_load_backend_submit_duration_ms`;分配 Cache buffer 并向下层 Store 提交 Load 的耗时。代码位置:`ucm/store/cache/cc/load_queue.cc` / `LoadQueue::DispatchOneTask` / shard 分发完成后。\n- `backend wait`:指标 `ucm:cache_shard_backend_wait_ms`;单个 Load shard 等待 Cache buffer 或下层 Load 任务就绪的时间。代码位置:`ucm/store/cache/cc/load_queue.cc` / `LoadQueue::TransferOneTask` / `WaitBackendTaskReady` 返回后。\n- `H2D sync`:指标 `ucm:cache_h2d_sync_ms`;最后一组 shard 的 H2D stream 同步耗时。代码位置:`ucm/store/cache/cc/load_queue.cc` / `LoadQueue::RecordH2dSyncMetrics` / stream 同步完成后。\n- `h2d submit & other`:计算方法 `Load 总耗时 - queue wait - backend submit - backend wait - H2D sync`;总耗时扣除其他分段平均耗时后的估算值。总耗时代码位置:`ucm/store/cache/cc/trans_manager.h` / `TransManager::Dispatch` / waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 45, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 44, + "w": 12, + "h": 8 + }, + "id": 58, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_queue_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_queue_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))", + "legendFormat": "queue wait {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_backend_submit_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_backend_submit_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))", + "legendFormat": "backend submit {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_shard_backend_wait_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n /\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_shard_backend_wait_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n)\nor\n(\n 0 * sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n)", + "legendFormat": "backend wait {{worker_id}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "clamp_min(\n (\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n /\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n )\n -\n (\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_queue_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n /\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_queue_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n )\n -\n (\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_backend_submit_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n /\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_backend_submit_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n )\n -\n (\n (\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_shard_backend_wait_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n /\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_shard_backend_wait_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n )\n or\n (\n 0 * sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n )\n )\n -\n (\n (\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_h2d_sync_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n /\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_h2d_sync_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n )\n or\n (\n 0 * sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n )\n ),\n 0\n)", + "legendFormat": "h2d submit & other {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_h2d_sync_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_h2d_sync_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n)\nor\n(\n 0 * sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n)", + "legendFormat": "H2D sync {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Load Avg Breakdown", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标与代码位置:\n- `ucm:cache_dump_queue_wait_duration_ms`:Dump 任务从提交到调度线程取出的等待时间。位置:`ucm/store/cache/cc/dump_queue.cc` / `DumpQueue::DispatchOneTask` / 开始处理任务时。\n- `ucm:cache_dump_mkbuf_duration_ms`:准备 Cache buffer 并提交 D2H 异步拷贝的耗时。位置:`ucm/store/cache/cc/dump_queue.cc` / `DumpQueue::DumpOneTask` / buffer 准备完成后。\n- `ucm:cache_d2h_duration_ms`:包含前置计算等待的 D2H stream 同步耗时。位置:`ucm/store/cache/cc/dump_queue.cc` / `DumpQueue::DumpOneTask` / stream 同步完成后。\n- `ucm:cache_dump_backend_submit_duration_ms`:向下层 Store 提交 Dump 的同步耗时。位置:`ucm/store/cache/cc/dump_queue.cc` / `DumpQueue::DumpOneTask` / `backend_->Dump` 返回后。\n- `ucm:cache_dump_duration_ms`:Dump 任务端到端耗时。位置:`ucm/store/cache/cc/trans_manager.h` / `TransManager::Dispatch` / waiter 完成回调。\n`other` 为总耗时减去前四项平均耗时后的估算值。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 45, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 44, + "w": 12, + "h": 8 + }, + "id": 59, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_queue_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_queue_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))", + "legendFormat": "queue wait {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_mkbuf_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_mkbuf_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))", + "legendFormat": "mkbuf {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_d2h_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_d2h_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))", + "legendFormat": "D2H sync (include wait compute) {{worker_id}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_backend_submit_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_backend_submit_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))", + "legendFormat": "backend submit {{worker_id}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "clamp_min(\n (\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n /\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n )\n -\n (\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_queue_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n /\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_queue_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n )\n -\n (\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_mkbuf_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n /\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_mkbuf_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n )\n -\n (\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_d2h_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n /\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_d2h_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n )\n -\n (\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_backend_submit_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n /\n sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_backend_submit_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[40s]))\n ),\n 0\n)", + "legendFormat": "other {{worker_id}}", + "range": true, + "refId": "F" + } + ], + "title": "Cache Dump Avg Breakdown", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_load_bytes_total` 表示 Cache Load 任务累计传输字节数;本面板用字节增速计算每个 worker 的聚合 GB/s,包含空闲间隔。代码位置:`ucm/store/cache/cc/trans_manager.h` / `TransManager::Dispatch` / Load waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: GB/s", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 52, + "w": 12, + "h": 8 + }, + "id": 60, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_bytes_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / 1e9", + "legendFormat": " {{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Load Bandwidth (aggregated)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_dump_bytes_total` 表示 Cache Dump 任务累计传输字节数;本面板用字节增速计算每个 worker 的聚合 GB/s,包含空闲间隔。代码位置:`ucm/store/cache/cc/trans_manager.h` / `TransManager::Dispatch` / Dump waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: GB/s", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 52, + "w": 12, + "h": 8 + }, + "id": 61, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_bytes_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / 1e9", + "legendFormat": " {{worker_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Cache Dump Bandwidth (aggregated)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_load_queue_wait_duration_ms` 表示 Cache Load 任务从提交到调度线程取出的排队时间,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/cache/cc/load_queue.cc` / `LoadQueue::DispatchOneTask` / 开始处理任务时。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 60, + "w": 12, + "h": 8 + }, + "id": 62, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_load_queue_wait_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_load_queue_wait_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_load_queue_wait_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_queue_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_queue_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Load Queue Wait Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_dump_queue_wait_duration_ms` 表示 Cache Dump 任务从提交到调度线程取出的排队时间,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/cache/cc/dump_queue.cc` / `DumpQueue::DispatchOneTask` / 开始处理任务时。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 12, + "y": 60, + "w": 12, + "h": 8 + }, + "id": 63, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_queue_wait_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_queue_wait_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_queue_wait_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_queue_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_queue_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Dump Queue Wait Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_dump_prereq_wait_ms` 表示 Dump 在开始 D2H 前等待前置计算事件完成的时间,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/cache/cc/dump_queue.cc` / `DumpQueue::DumpOneTask` / 前置事件回调完成后。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 68, + "w": 12, + "h": 8 + }, + "id": 64, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_prereq_wait_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_prereq_wait_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_prereq_wait_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_prereq_wait_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_prereq_wait_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Dump Wait Compute Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_h2d_sync_ms` 表示最后一组 Cache Load shard 提交后等待 H2D stream 完成的时间,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/cache/cc/load_queue.cc` / `LoadQueue::RecordH2dSyncMetrics` / stream 同步完成后。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 12, + "y": 68, + "w": 12, + "h": 8 + }, + "id": 65, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_h2d_sync_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_h2d_sync_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_h2d_sync_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_h2d_sync_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_h2d_sync_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Load H2D Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_d2h_duration_ms` 表示 Cache Dump 的 D2H stream 同步时间,其中包含前置计算等待,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/cache/cc/dump_queue.cc` / `DumpQueue::DumpOneTask` / stream 同步完成后。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 76, + "w": 12, + "h": 8 + }, + "id": 66, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_d2h_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_d2h_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_d2h_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_d2h_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_d2h_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Dump D2H Duration (include wait compute)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_load_backend_submit_duration_ms` 表示分配 Cache buffer 并向下层 Store 同步提交 Load 的耗时,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/cache/cc/load_queue.cc` / `LoadQueue::DispatchOneTask` / shard 分发完成后。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 12, + "y": 76, + "w": 12, + "h": 8 + }, + "id": 67, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_load_backend_submit_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_load_backend_submit_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_load_backend_submit_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_backend_submit_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_backend_submit_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Load Backend Submit Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_dump_mkbuf_duration_ms` 表示准备 Cache buffer 并提交 D2H 异步拷贝的耗时,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/cache/cc/dump_queue.cc` / `DumpQueue::DumpOneTask` / buffer 准备完成后。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 84, + "w": 12, + "h": 8 + }, + "id": 68, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_mkbuf_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_mkbuf_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_mkbuf_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_mkbuf_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_mkbuf_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Dump MkBuf Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_h2d_submit_ms` 表示单个 shard 在下层数据就绪后提交 H2D 异步拷贝的 CPU 耗时,不包含实际传输和 stream 同步时间。代码位置:`ucm/store/cache/cc/load_queue.cc` / `LoadQueue::TransferOneTask` / `HostToDeviceAsync` 返回后。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 12, + "y": 84, + "w": 12, + "h": 8 + }, + "id": 69, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_h2d_submit_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_h2d_submit_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_h2d_submit_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_h2d_submit_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_h2d_submit_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Load H2D Submit Cost", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_dump_backend_submit_duration_ms` 表示向下层 Store 同步提交 Dump 的耗时,不包含下层写入完成时间。代码位置:`ucm/store/cache/cc/dump_queue.cc` / `DumpQueue::DumpOneTask` / `backend_->Dump` 返回后。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 92, + "w": 12, + "h": 8 + }, + "id": 70, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_backend_submit_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_backend_submit_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_backend_submit_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_backend_submit_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_backend_submit_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Dump Backend Submit Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_load_bandwidth_gbps` 表示单个 Cache Load 任务的字节数除以该任务端到端耗时所得带宽,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/cache/cc/trans_manager.h` / `TransManager::Dispatch` / Load waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: GB/s", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 12, + "y": 92, + "w": 12, + "h": 8 + }, + "id": 71, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_load_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_load_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_load_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_bandwidth_gbps_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_load_bandwidth_gbps_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Load Bandwidth (per task)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_dump_bandwidth_gbps` 表示单个 Cache Dump 任务的字节数除以该任务端到端耗时所得带宽,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/cache/cc/trans_manager.h` / `TransManager::Dispatch` / Dump waiter 完成回调。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "suffix: GB/s", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 100, + "w": 12, + "h": 8 + }, + "id": 72, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_bandwidth_gbps_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_bandwidth_gbps_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Dump Bandwidth (per task)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_shard_backend_wait_ms` 表示单个 Cache Load shard 等待 buffer 或下层 Load 任务就绪的时间,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/cache/cc/load_queue.cc` / `LoadQueue::TransferOneTask` / `WaitBackendTaskReady` 返回后。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 12, + "y": 100, + "w": 12, + "h": 8 + }, + "id": 73, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_shard_backend_wait_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_shard_backend_wait_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_shard_backend_wait_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_shard_backend_wait_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_shard_backend_wait_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Load Backend Wait Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_lookup_duration_ms` 表示 Cache 内存 buffer 扫描的耗时,本面板展示每次 Lookup 调用的 p50、p90、p99 和平均值。未开启共享内存时不会查询 Cache buffer,请求会直接查询下层 Store,因此该指标不会产生数据。代码位置:`ucm/store/cache/cc/buffer_manager.h` / `BufferManager::Lookup`(私有快速扫描函数)/ 扫描完成后。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 108, + "w": 12, + "h": 8 + }, + "id": 74, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_lookup_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_lookup_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_lookup_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_lookup_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_lookup_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Lookup Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_lookup_backend_duration_ms` 表示 Cache 未命中或未启用 buffer 时查询下层 Store 的耗时,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/cache/cc/buffer_manager.h` / `BufferManager::LookupThrough`、`LookupFast`、`LookupOnPrefixFast`、`LookupOnReverseFast` / 下层 Lookup 返回后。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 12, + "y": 108, + "w": 12, + "h": 8 + }, + "id": 75, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_lookup_backend_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_lookup_backend_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_lookup_backend_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_lookup_backend_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_lookup_backend_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Lookup Backend Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标:`ucm:cache_dump_backend_wait_duration_ms` 表示等待下层 Store 完成 Cache Dump 写入的时间,本面板展示 p50、p90、p99 和平均值。代码位置:`ucm/store/cache/cc/dump_queue.cc` / `DumpQueue::BackendDumpStage` / `backend_->Wait` 返回后。", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ms", + "noValue": "No data" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "^p50.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p90.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 6 + ] + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^p99.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 4, + 4 + ] + } + }, + { + "id": "custom.lineWidth", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "^avg.*" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "solid" + } + }, + { + "id": "custom.lineWidth", + "value": 1 + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "x": 0, + "y": 116, + "w": 12, + "h": 8 + }, + "id": 76, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "maxHeight": "50%", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_backend_wait_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_backend_wait_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}cache_dump_backend_wait_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_backend_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))\n/\nsum by (${perWorker:raw}) (rate(${ucm_prefix}cache_dump_backend_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "avg {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Cache Dump Backend Wait Duration", + "type": "timeseries" + }, + { + "type": "heatmap", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 77, + "title": "Cache / Load Duration", + "description": "指标:`ucm:cache_load_duration_ms` 表示 Cache Load 任务端到端耗时;热力图使用 bucket 增速显示各耗时区间的任务速率。代码位置:`ucm/store/cache/cc/trans_manager.h` / `TransManager::Dispatch` / Load waiter 完成回调。", + "gridPos": { + "x": 12, + "y": 116, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (le) (rate(${ucm_prefix}cache_load_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "options": { + "calculate": false, + "yAxis": { + "axisPlacement": "left", + "unit": "ms", + "decimals": 0, + "reverse": false + }, + "rowsFrame": { + "layout": "auto", + "value": "Request rate" + }, + "color": { + "mode": "scheme", + "scheme": "Spectral", + "fill": "dark-orange", + "exponent": 0.5, + "steps": 64, + "reverse": false + }, + "cellGap": 1, + "cellValues": { + "unit": "suffix: req/s" + }, + "filterValues": { + "le": 1e-09 + }, + "tooltip": { + "show": true, + "yHistogram": true, + "mode": "single" + }, + "legend": { + "show": true + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "noValue": "No data" + }, + "overrides": [] + }, + "pluginVersion": "10.4.0" + }, + { + "type": "heatmap", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 78, + "title": "Cache / Load Bandwidth (per task)", + "description": "指标:`ucm:cache_load_bandwidth_gbps` 表示单个 Cache Load 任务的任务级带宽;热力图使用 bucket 增速显示各带宽区间的任务速率。代码位置:`ucm/store/cache/cc/trans_manager.h` / `TransManager::Dispatch` / Load waiter 完成回调。", + "gridPos": { + "x": 0, + "y": 124, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (le) (rate(${ucm_prefix}cache_load_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "options": { + "calculate": false, + "yAxis": { + "axisPlacement": "left", + "unit": "Bps", + "decimals": 0, + "reverse": false + }, + "rowsFrame": { + "layout": "auto", + "value": "Request rate" + }, + "color": { + "mode": "scheme", + "scheme": "Spectral", + "fill": "dark-orange", + "exponent": 0.5, + "steps": 64, + "reverse": false + }, + "cellGap": 1, + "cellValues": { + "unit": "suffix: req/s" + }, + "filterValues": { + "le": 1e-09 + }, + "tooltip": { + "show": true, + "yHistogram": true, + "mode": "single" + }, + "legend": { + "show": true + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "noValue": "No data", + "unit": "suffix: GB/s" + }, + "overrides": [] + }, + "pluginVersion": "10.4.0" + }, + { + "type": "heatmap", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 79, + "title": "Cache / Dump Duration", + "description": "指标:`ucm:cache_dump_duration_ms` 表示 Cache Dump 任务端到端耗时;热力图使用 bucket 增速显示各耗时区间的任务速率。代码位置:`ucm/store/cache/cc/trans_manager.h` / `TransManager::Dispatch` / Dump waiter 完成回调。", + "gridPos": { + "x": 12, + "y": 124, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (le) (rate(${ucm_prefix}cache_dump_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "options": { + "calculate": false, + "yAxis": { + "axisPlacement": "left", + "unit": "ms", + "decimals": 0, + "reverse": false + }, + "rowsFrame": { + "layout": "auto", + "value": "Request rate" + }, + "color": { + "mode": "scheme", + "scheme": "Spectral", + "fill": "dark-orange", + "exponent": 0.5, + "steps": 64, + "reverse": false + }, + "cellGap": 1, + "cellValues": { + "unit": "suffix: req/s" + }, + "filterValues": { + "le": 1e-09 + }, + "tooltip": { + "show": true, + "yHistogram": true, + "mode": "single" + }, + "legend": { + "show": true + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "noValue": "No data" + }, + "overrides": [] + }, + "pluginVersion": "10.4.0" + }, + { + "type": "heatmap", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "id": 80, + "title": "Cache / Dump Bandwidth (per task)", + "description": "指标:`ucm:cache_dump_bandwidth_gbps` 表示单个 Cache Dump 任务的任务级带宽;热力图使用 bucket 增速显示各带宽区间的任务速率。代码位置:`ucm/store/cache/cc/trans_manager.h` / `TransManager::Dispatch` / Dump waiter 完成回调。", + "gridPos": { + "x": 0, + "y": 132, + "w": 12, + "h": 8 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (le) (rate(${ucm_prefix}cache_dump_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "format": "heatmap", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "options": { + "calculate": false, + "yAxis": { + "axisPlacement": "left", + "unit": "Bps", + "decimals": 0, + "reverse": false + }, + "rowsFrame": { + "layout": "auto", + "value": "Request rate" + }, + "color": { + "mode": "scheme", + "scheme": "Spectral", + "fill": "dark-orange", + "exponent": 0.5, + "steps": 64, + "reverse": false + }, + "cellGap": 1, + "cellValues": { + "unit": "suffix: req/s" + }, + "filterValues": { + "le": 1e-09 + }, + "tooltip": { + "show": true, + "yHistogram": true, + "mode": "single" + }, + "legend": { + "show": true + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "noValue": "No data", + "unit": "suffix: GB/s" + }, + "overrides": [] + }, + "pluginVersion": "10.4.0" + }, + { + "title": "cache_lookup_hit_blocks_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of lookup hits served by the Cache stage (no descent to backend)\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_lookup_hit_blocks_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 81, + "gridPos": { + "x": 12, + "y": 132, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_lookup_miss_blocks_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of lookup misses at the Cache stage (had to query the backend)\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_lookup_miss_blocks_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 82, + "gridPos": { + "x": 0, + "y": 140, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_load_shards_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total shards whose Cache buffer state was inspected during load\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_load_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 83, + "gridPos": { + "x": 12, + "y": 140, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_load_wait_shards_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shards whose Cache buffer was not ready when acquired and required waiting\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_load_wait_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 84, + "gridPos": { + "x": 0, + "y": 148, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_load_backend_shards_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shards that descended to the backend on load (true cache miss at the buffer-allocation stage; aka backend-load count)\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_load_backend_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 85, + "gridPos": { + "x": 12, + "y": 148, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_load_success_shards_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shards successfully loaded from an already-ready Cache buffer to device\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_load_success_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 86, + "gridPos": { + "x": 0, + "y": 156, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_posix_load_success_shards_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shards successfully loaded to device after waiting for Posix to fill Cache\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_posix_load_success_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 87, + "gridPos": { + "x": 12, + "y": 156, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_dump_shards_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total shard descriptors processed by Cache dump, including failed tasks\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_dump_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 88, + "gridPos": { + "x": 0, + "y": 164, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_dump_backend_shards_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shards actually pushed to backend on dump (excludes !handle.Owner() skips in shared-buffer scenario)\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_dump_backend_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 89, + "gridPos": { + "x": 12, + "y": 164, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_load_queue_full_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Cache load submissions rejected because the waiting queue was full\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_load_queue_full_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 90, + "gridPos": { + "x": 0, + "y": 172, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_dump_queue_full_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Cache dump submissions rejected because the waiting queue was full\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_dump_queue_full_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 91, + "gridPos": { + "x": 12, + "y": 172, + "w": 12, + "h": 8 + } + } + ], + "id": 53, + "gridPos": { + "x": 0, + "y": 27, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Cache / Errors and recovery", + "collapsed": true, + "panels": [ + { + "title": "cache_load_failed_shards_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Cache load shards that did not complete device delivery\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_load_failed_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 93, + "gridPos": { + "x": 0, + "y": 29, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_backend_load_submit_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Cache load backend submit failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_backend_load_submit_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 94, + "gridPos": { + "x": 12, + "y": 29, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_backend_load_wait_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Cache load backend wait failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_backend_load_wait_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 95, + "gridPos": { + "x": 0, + "y": 37, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_backend_dump_submit_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Cache dump backend submit failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_backend_dump_submit_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 96, + "gridPos": { + "x": 12, + "y": 37, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_backend_dump_wait_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Cache dump backend wait failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_backend_dump_wait_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 97, + "gridPos": { + "x": 0, + "y": 45, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_h2d_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Cache host-to-device transfer or sync failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_h2d_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 98, + "gridPos": { + "x": 12, + "y": 45, + "w": 12, + "h": 8 + } + }, + { + "title": "cache_d2h_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Cache device-to-host transfer, event wait, or sync failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}cache_d2h_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 99, + "gridPos": { + "x": 0, + "y": 53, + "w": 12, + "h": 8 + } + } + ], + "id": 92, + "gridPos": { + "x": 0, + "y": 28, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Cache / Cumulative counters and observations", + "collapsed": true, + "panels": [ + { + "title": "Cache / cumulative counter samples", + "type": "table", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Cumulative values since worker start. Histogram count is the number of observations, not a success count.", + "targets": [ + { + "expr": "{__name__=~\"${ucm_prefix}(cache_lookup_hit_blocks_total|cache_lookup_miss_blocks_total|cache_load_shards_total|cache_load_wait_shards_total|cache_load_backend_shards_total|cache_load_success_shards_total|cache_posix_load_success_shards_total|cache_load_failed_shards_total|cache_dump_shards_total|cache_dump_backend_shards_total|cache_load_queue_full_total|cache_dump_queue_full_total|cache_backend_load_submit_errors_total|cache_backend_load_wait_errors_total|cache_backend_dump_submit_errors_total|cache_backend_dump_wait_errors_total|cache_h2d_errors_total|cache_d2h_errors_total|cache_load_bytes_total|cache_dump_bytes_total)\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": true, + "range": false, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "id": 101, + "gridPos": { + "x": 0, + "y": 30, + "w": 24, + "h": 8 + } + }, + { + "title": "Cache / cumulative histogram samples", + "type": "table", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Cumulative values since worker start. Histogram count is the number of observations, not a success count.", + "targets": [ + { + "expr": "{__name__=~\"${ucm_prefix}(cache_lookup_duration_ms_count|cache_lookup_backend_duration_ms_count|cache_load_duration_ms_count|cache_dump_duration_ms_count|cache_load_bandwidth_gbps_count|cache_dump_bandwidth_gbps_count|cache_load_queue_wait_duration_ms_count|cache_dump_queue_wait_duration_ms_count|cache_load_backend_submit_duration_ms_count|cache_shard_backend_wait_ms_count|cache_h2d_submit_ms_count|cache_h2d_sync_ms_count|cache_dump_mkbuf_duration_ms_count|cache_dump_prereq_wait_ms_count|cache_d2h_duration_ms_count|cache_dump_backend_submit_duration_ms_count|cache_dump_backend_wait_duration_ms_count)\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": true, + "range": false, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "id": 102, + "gridPos": { + "x": 0, + "y": 38, + "w": 24, + "h": 8 + } + } + ], + "id": 100, + "gridPos": { + "x": 0, + "y": 29, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Mooncake / Health and resources", + "collapsed": true, + "panels": [ + { + "title": "mooncake_healthy_count_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of successful Mooncake health probes\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}mooncake_healthy_count_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 104, + "gridPos": { + "x": 0, + "y": 31, + "w": 12, + "h": 8 + } + }, + { + "title": "mooncake_store_health", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Effective Mooncake health breaker state, where 1 is enabled and 0 is fused\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}mooncake_store_health{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + }, + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "Fused", + "color": "red" + }, + "1": { + "text": "Enabled", + "color": "green" + } + } + } + ] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 105, + "gridPos": { + "x": 12, + "y": 31, + "w": 12, + "h": 8 + } + } + ], + "id": 103, + "gridPos": { + "x": 0, + "y": 30, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Mooncake / Throughput and latency", + "collapsed": true, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标与代码位置:\n- `ucm:mooncake_load_blocks_total`:累计完成加载的 block 数。位置:`ucm/store/mooncakestore/cc/trans_manager.cc` / `TransManager::Dispatch` / Load waiter 完成回调。\n- `ucm:mooncake_dump_blocks_total`:累计完成写出的 block 数。位置:`ucm/store/mooncakestore/cc/trans_manager.cc` / `TransManager::Dispatch` / Dump waiter 完成回调。\n本面板展示两个 Counter 的每秒增速。", + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 32, + "w": 12, + "h": 8 + }, + "id": 107, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_blocks_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load blocks {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_blocks_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump blocks {{worker_id}}", + "range": true, + "refId": "B" + } + ], + "title": "Mooncake Block Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标与代码位置:\n- `ucm:mooncake_load_bytes_total`:Mooncake Load 任务累计逻辑字节数。位置:`ucm/store/mooncakestore/cc/trans_manager.cc` / `TransManager::Dispatch` / Load waiter 完成回调。\n- `ucm:mooncake_dump_bytes_total`:Mooncake Dump 任务累计逻辑字节数。位置:`ucm/store/mooncakestore/cc/trans_manager.cc` / `TransManager::Dispatch` / Dump waiter 完成回调。\n- `ucm:mooncake_h2d_bytes_total`:累计 H2D 拷贝字节数。位置:`ucm/store/mooncakestore/cc/load_queue.cc` / `LoadQueue::RecordH2dMetrics`,以及 `ucm/store/mooncakestore/cc/share_load_queue.cc` / `ShareLoadQueue::RecordH2dMetrics` / stream 同步完成后。\n- `ucm:mooncake_d2h_bytes_total`:累计 D2H 拷贝字节数。位置:`ucm/store/mooncakestore/cc/dump_queue.cc` / `DumpQueue::RecordD2hMetrics` / stream 同步完成后。\n本面板展示各 Counter 的每秒字节增速。", + "fieldConfig": { + "defaults": { + "unit": "Bps", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 32, + "w": 12, + "h": 8 + }, + "id": 108, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_bytes_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "load bytes {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_bytes_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "dump bytes {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_h2d_bytes_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "h2d bytes {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_d2h_bytes_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "d2h bytes {{worker_id}}", + "range": true, + "refId": "D" + } + ], + "title": "Mooncake Byte Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标与代码位置:\n- `ucm:mooncake_load_hit_shards_total`:Mooncake 直接加载命中的 shard 数。位置:`ucm/store/mooncakestore/cc/load_queue.cc` / `LoadQueue::DispatchOneTask` / `batch_get` 结果统计后。\n- `ucm:mooncake_load_miss_shards_total`:Mooncake 加载未命中的 shard 数。位置:`ucm/store/mooncakestore/cc/load_queue.cc` / `LoadQueue::DispatchOneTask` / `batch_get` 结果统计后。\n- `ucm:mooncake_load_backend_shards_total`:未命中后提交给下层 Store 的 shard 数。位置:`ucm/store/mooncakestore/cc/load_queue.cc` / `LoadQueue::SubmitMissShards`,以及 `ucm/store/mooncakestore/cc/share_load_queue.cc` / `ShareLoadQueue::HandleLoadTask` / 下层 Load 提交成功后。\n本面板展示各 Counter 的每秒增速。", + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 40, + "w": 12, + "h": 8 + }, + "id": 109, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_hit_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "hit {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_miss_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "miss {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_backend_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "backend {{worker_id}}", + "range": true, + "refId": "C" + } + ], + "title": "Mooncake Load Hit / Miss Shards", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标与代码位置:\n- `ucm:mooncake_dump_existing_shards_total`:Dump 前检查到已存在于 Mooncake 的 shard 数。位置:`ucm/store/mooncakestore/cc/dump_queue.cc` / `DumpQueue::DumpOneTask` / `batchIsExist` 结果统计后。\n- `ucm:mooncake_dump_missing_shards_total`:Dump 前检查到 Mooncake 中不存在的 shard 数。位置:`ucm/store/mooncakestore/cc/dump_queue.cc` / `DumpQueue::DumpOneTask` / `batchIsExist` 结果统计后。\n- `ucm:mooncake_dump_backend_shards_total`:同时提交给下层 Store 的 shard 数。位置:`ucm/store/mooncakestore/cc/dump_queue.cc` / `DumpQueue::SubmitBackendDump` / 下层 Dump 提交成功后。\n本面板展示各 Counter 的每秒增速。", + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 40, + "w": 12, + "h": 8 + }, + "id": 110, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_existing_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "existing {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_missing_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "missing {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_backend_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "backend {{worker_id}}", + "range": true, + "refId": "C" + } + ], + "title": "Mooncake Dump Existing / Missing Shards", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标与代码位置:\n- `ucm:mooncake_load_duration_ms`:Mooncake Load 任务端到端耗时。位置:`ucm/store/mooncakestore/cc/trans_manager.cc` / `TransManager::Dispatch` / Load waiter 完成回调。\n- `ucm:mooncake_dump_duration_ms`:Mooncake Dump 任务端到端耗时。位置:`ucm/store/mooncakestore/cc/trans_manager.cc` / `TransManager::Dispatch` / Dump waiter 完成回调。\n本面板展示两项指标的 p50、p90、p99 和平均值。", + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 48, + "w": 12, + "h": 8 + }, + "id": 111, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "load p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "load p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "load p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "load avg {{worker_id}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "dump p50 {{worker_id}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "dump p90 {{worker_id}}", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_duration_ms_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "dump p99 {{worker_id}}", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "dump avg {{worker_id}}", + "range": true, + "refId": "H" + } + ], + "title": "Mooncake Load / Dump Duration", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标与代码位置:\n- `ucm:mooncake_load_bandwidth_gbps`:单个 Mooncake Load 任务的字节数除以端到端耗时所得带宽。位置:`ucm/store/mooncakestore/cc/trans_manager.cc` / `TransManager::Dispatch` / Load waiter 完成回调。\n- `ucm:mooncake_dump_bandwidth_gbps`:单个 Mooncake Dump 任务的字节数除以端到端耗时所得带宽。位置:`ucm/store/mooncakestore/cc/trans_manager.cc` / `TransManager::Dispatch` / Dump waiter 完成回调。\n本面板展示两项指标的 p50、p90、p99 和平均值。", + "fieldConfig": { + "defaults": { + "unit": "suffix: GB/s", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 48, + "w": 12, + "h": 8 + }, + "id": 112, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "load p50 {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "load p90 {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "load p99 {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_bandwidth_gbps_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_bandwidth_gbps_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "load avg {{worker_id}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "dump p50 {{worker_id}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.9, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "dump p90 {{worker_id}}", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, sum by (le, ${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_bandwidth_gbps_bucket{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])))", + "legendFormat": "dump p99 {{worker_id}}", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_bandwidth_gbps_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_bandwidth_gbps_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "dump avg {{worker_id}}", + "range": true, + "refId": "H" + } + ], + "title": "Mooncake Load / Dump Bandwidth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标与代码位置:\n- `ucm:mooncake_load_queue_wait_duration_ms`:Load 任务排队时间。位置:`ucm/store/mooncakestore/cc/load_queue.cc` / `LoadQueue::DispatchOneTask` / 开始处理任务时。\n- `ucm:mooncake_get_duration_ms`:调用 Mooncake `batch_get` 的耗时。位置:`ucm/store/mooncakestore/cc/load_queue.cc` / `LoadQueue::TryMooncakeLoad` / 调用返回后。\n- `ucm:mooncake_load_backend_submit_duration_ms`:向下层 Store 提交 Load 的耗时。位置:`ucm/store/mooncakestore/cc/load_queue.cc` / `LoadQueue::SubmitMissShards`,以及 `ucm/store/mooncakestore/cc/share_load_queue.cc` / `ShareLoadQueue::HandleLoadTask` / 下层 Load 返回后。\n- `ucm:mooncake_backend_load_wait_duration_ms`:等待下层 Load 完成的耗时。位置:`ucm/store/mooncakestore/cc/load_queue.cc` / `LoadQueue::TransferOneTask`,以及 `ucm/store/mooncakestore/cc/share_load_queue.cc` / `ShareLoadQueue::HandleBackendComplete` / `backend_->Wait` 返回后。\n- `ucm:mooncake_h2d_duration_ms`:H2D stream 同步耗时。位置:`ucm/store/mooncakestore/cc/load_queue.cc` / `LoadQueue::RecordH2dMetrics`,以及 `ucm/store/mooncakestore/cc/share_load_queue.cc` / `ShareLoadQueue::RecordH2dMetrics` / stream 同步完成后。", + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 56, + "w": 12, + "h": 8 + }, + "id": 113, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_queue_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_queue_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "queue wait {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_get_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_get_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "get {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_backend_submit_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_load_backend_submit_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "backend submit {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_backend_load_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_backend_load_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "backend wait {{worker_id}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_h2d_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_h2d_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "h2d sync {{worker_id}}", + "range": true, + "refId": "E" + } + ], + "title": "Mooncake Load Stage Avg Breakdown", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "指标与代码位置:\n- `ucm:mooncake_dump_queue_wait_duration_ms`:Dump 任务排队时间。位置:`ucm/store/mooncakestore/cc/dump_queue.cc` / `DumpQueue::DispatchOneTask` / 开始处理任务时。\n- `ucm:mooncake_exists_duration_ms`:调用 Mooncake `batchIsExist` 的耗时。位置:`ucm/store/mooncakestore/cc/dump_queue.cc` / `DumpQueue::DumpOneTask` / 调用返回后。\n- `ucm:mooncake_dump_prereq_wait_ms`:等待前置计算事件的耗时。位置:`ucm/store/mooncakestore/cc/dump_queue.cc` / `DumpQueue::WaitPrerequisite` / 事件同步返回后。\n- `ucm:mooncake_d2h_duration_ms`:D2H stream 同步耗时。位置:`ucm/store/mooncakestore/cc/dump_queue.cc` / `DumpQueue::RecordD2hMetrics` / stream 同步完成后。\n- `ucm:mooncake_put_duration_ms`:调用 Mooncake `batch_put` 的耗时。位置:`ucm/store/mooncakestore/cc/dump_queue.cc` / `DumpQueue::PutToMooncake` / 调用返回后。\n- `ucm:mooncake_dump_backend_submit_duration_ms`:向下层 Store 提交 Dump 的耗时。位置:`ucm/store/mooncakestore/cc/dump_queue.cc` / `DumpQueue::SubmitBackendDump` / `backend_->Dump` 返回后。\n- `ucm:mooncake_dump_backend_wait_duration_ms`:等待下层 Dump 完成的耗时。位置:`ucm/store/mooncakestore/cc/dump_queue.cc` / `DumpQueue::BackendDumpStage` / `backend_->Wait` 返回后。", + "fieldConfig": { + "defaults": { + "unit": "ms", + "noValue": "No data" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 56, + "w": 12, + "h": 8 + }, + "id": 114, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_queue_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_queue_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "queue wait {{worker_id}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_exists_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_exists_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "exists {{worker_id}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_prereq_wait_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_prereq_wait_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "compute wait {{worker_id}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_put_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_put_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "put {{worker_id}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_d2h_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_d2h_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "d2h sync {{worker_id}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_backend_submit_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_backend_submit_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "backend submit {{worker_id}}", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_backend_wait_duration_ms_sum{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])) / clamp_min(sum by (${perWorker:raw}) (rate(${ucm_prefix}mooncake_dump_backend_wait_duration_ms_count{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval])), 1e-9)", + "legendFormat": "backend wait {{worker_id}}", + "range": true, + "refId": "G" + } + ], + "title": "Mooncake Dump Stage Avg Breakdown", + "type": "timeseries" + }, + { + "title": "mooncake_lookup_hit_blocks_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Blocks found directly by Mooncake lookup before backend descent\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}mooncake_lookup_hit_blocks_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 115, + "gridPos": { + "x": 0, + "y": 64, + "w": 12, + "h": 8 + } + }, + { + "title": "mooncake_load_queue_full_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Mooncake load submissions rejected because the waiting queue was full\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}mooncake_load_queue_full_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 116, + "gridPos": { + "x": 12, + "y": 64, + "w": 12, + "h": 8 + } + }, + { + "title": "mooncake_dump_queue_full_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Mooncake dump submissions rejected because the waiting queue was full\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}mooncake_dump_queue_full_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 117, + "gridPos": { + "x": 0, + "y": 72, + "w": 12, + "h": 8 + } + } + ], + "id": 106, + "gridPos": { + "x": 0, + "y": 31, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Mooncake / Errors and recovery", + "collapsed": true, + "panels": [ + { + "title": "mooncake_unhealthy_count_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of failed Mooncake health probes\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}mooncake_unhealthy_count_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 119, + "gridPos": { + "x": 0, + "y": 33, + "w": 12, + "h": 8 + } + }, + { + "title": "mooncake_passive_failures_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of failed Mooncake IO tasks observed by passive health detection\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}mooncake_passive_failures_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 120, + "gridPos": { + "x": 12, + "y": 33, + "w": 12, + "h": 8 + } + }, + { + "title": "mooncake_get_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Mooncake batch get failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}mooncake_get_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 121, + "gridPos": { + "x": 0, + "y": 41, + "w": 12, + "h": 8 + } + }, + { + "title": "mooncake_put_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Mooncake batch put failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}mooncake_put_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 122, + "gridPos": { + "x": 12, + "y": 41, + "w": 12, + "h": 8 + } + }, + { + "title": "mooncake_backend_load_submit_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Mooncake backend load submit failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}mooncake_backend_load_submit_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 123, + "gridPos": { + "x": 0, + "y": 49, + "w": 12, + "h": 8 + } + }, + { + "title": "mooncake_backend_load_wait_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Mooncake backend load wait failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}mooncake_backend_load_wait_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 124, + "gridPos": { + "x": 12, + "y": 49, + "w": 12, + "h": 8 + } + }, + { + "title": "mooncake_backend_dump_submit_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Mooncake backend dump submit failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}mooncake_backend_dump_submit_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 125, + "gridPos": { + "x": 0, + "y": 57, + "w": 12, + "h": 8 + } + }, + { + "title": "mooncake_backend_dump_wait_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Mooncake backend dump wait failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}mooncake_backend_dump_wait_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 126, + "gridPos": { + "x": 12, + "y": 57, + "w": 12, + "h": 8 + } + }, + { + "title": "mooncake_h2d_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Mooncake H2D transfer or sync failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}mooncake_h2d_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 127, + "gridPos": { + "x": 0, + "y": 65, + "w": 12, + "h": 8 + } + }, + { + "title": "mooncake_d2h_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of Mooncake D2H transfer, event wait, or sync failures\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}mooncake_d2h_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 128, + "gridPos": { + "x": 12, + "y": 65, + "w": 12, + "h": 8 + } + } + ], + "id": 118, + "gridPos": { + "x": 0, + "y": 32, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Mooncake / Cumulative counters and observations", + "collapsed": true, + "panels": [ + { + "title": "Mooncake / cumulative counter samples", + "type": "table", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Cumulative values since worker start. Histogram count is the number of observations, not a success count.", + "targets": [ + { + "expr": "{__name__=~\"${ucm_prefix}(mooncake_load_blocks_total|mooncake_dump_blocks_total|mooncake_lookup_hit_blocks_total|mooncake_healthy_count_total|mooncake_unhealthy_count_total|mooncake_passive_failures_total|mooncake_load_bytes_total|mooncake_dump_bytes_total|mooncake_load_hit_shards_total|mooncake_load_miss_shards_total|mooncake_load_backend_shards_total|mooncake_dump_existing_shards_total|mooncake_dump_missing_shards_total|mooncake_dump_backend_shards_total|mooncake_load_queue_full_total|mooncake_dump_queue_full_total|mooncake_get_errors_total|mooncake_put_errors_total|mooncake_backend_load_submit_errors_total|mooncake_backend_load_wait_errors_total|mooncake_backend_dump_submit_errors_total|mooncake_backend_dump_wait_errors_total|mooncake_h2d_errors_total|mooncake_d2h_errors_total|mooncake_h2d_bytes_total|mooncake_d2h_bytes_total)\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": true, + "range": false, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "id": 130, + "gridPos": { + "x": 0, + "y": 34, + "w": 24, + "h": 8 + } + }, + { + "title": "Mooncake / cumulative histogram samples", + "type": "table", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Cumulative values since worker start. Histogram count is the number of observations, not a success count.", + "targets": [ + { + "expr": "{__name__=~\"${ucm_prefix}(mooncake_load_duration_ms_count|mooncake_dump_duration_ms_count|mooncake_load_bandwidth_gbps_count|mooncake_dump_bandwidth_gbps_count|mooncake_load_queue_wait_duration_ms_count|mooncake_dump_queue_wait_duration_ms_count|mooncake_get_duration_ms_count|mooncake_exists_duration_ms_count|mooncake_put_duration_ms_count|mooncake_load_backend_submit_duration_ms_count|mooncake_backend_load_wait_duration_ms_count|mooncake_h2d_duration_ms_count|mooncake_dump_prereq_wait_ms_count|mooncake_d2h_duration_ms_count|mooncake_dump_backend_submit_duration_ms_count|mooncake_dump_backend_wait_duration_ms_count)\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": true, + "range": false, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "id": 131, + "gridPos": { + "x": 0, + "y": 42, + "w": 24, + "h": 8 + } + } + ], + "id": 129, + "gridPos": { + "x": 0, + "y": 33, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Yuanrong / Health and resources", + "collapsed": true, + "panels": [ + { + "title": "yuanrong_dram_used_bytes", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "YuanRong physical shared-memory usage in bytes\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}yuanrong_dram_used_bytes{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 133, + "gridPos": { + "x": 0, + "y": 35, + "w": 12, + "h": 8 + } + }, + { + "title": "yuanrong_dram_capacity_bytes", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "YuanRong shared-memory capacity in bytes\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}yuanrong_dram_capacity_bytes{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 134, + "gridPos": { + "x": 12, + "y": 35, + "w": 12, + "h": 8 + } + }, + { + "title": "yuanrong_dram_usage_ratio", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "YuanRong physical shared-memory usage ratio\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}yuanrong_dram_usage_ratio{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 135, + "gridPos": { + "x": 0, + "y": 43, + "w": 12, + "h": 8 + } + }, + { + "title": "yuanrong_ssd_used_bytes", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "YuanRong physical spill-disk usage in bytes\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}yuanrong_ssd_used_bytes{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 136, + "gridPos": { + "x": 12, + "y": 43, + "w": 12, + "h": 8 + } + }, + { + "title": "yuanrong_ssd_capacity_bytes", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "YuanRong spill-disk capacity in bytes\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}yuanrong_ssd_capacity_bytes{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 137, + "gridPos": { + "x": 0, + "y": 51, + "w": 12, + "h": 8 + } + }, + { + "title": "yuanrong_ssd_usage_ratio", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "YuanRong physical spill-disk usage ratio\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}yuanrong_ssd_usage_ratio{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 138, + "gridPos": { + "x": 12, + "y": 51, + "w": 12, + "h": 8 + } + } + ], + "id": 132, + "gridPos": { + "x": 0, + "y": 34, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Yuanrong / Throughput and latency", + "collapsed": true, + "panels": [ + { + "title": "yuanrong_load_success_shards_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shards successfully loaded from YuanRong to device\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}yuanrong_load_success_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 140, + "gridPos": { + "x": 0, + "y": 36, + "w": 12, + "h": 8 + } + }, + { + "title": "yuanrong_lookup_miss_posix_load_success_shards_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shards successfully loaded from Posix after YuanRong lookup miss\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}yuanrong_lookup_miss_posix_load_success_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 141, + "gridPos": { + "x": 12, + "y": 36, + "w": 12, + "h": 8 + } + }, + { + "title": "yuanrong_load_fallback_posix_load_success_shards_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Shards successfully loaded from Posix after YuanRong load failure\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}yuanrong_load_fallback_posix_load_success_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 142, + "gridPos": { + "x": 0, + "y": 44, + "w": 12, + "h": 8 + } + }, + { + "title": "yuanrong_local_dram_load_hits_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Estimated YuanRong local DRAM Get hits forwarded from kv_resource.log\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}yuanrong_local_dram_load_hits_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 143, + "gridPos": { + "x": 12, + "y": 44, + "w": 12, + "h": 8 + } + }, + { + "title": "yuanrong_remote_load_hits_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Estimated YuanRong remote worker Get hits forwarded from kv_resource.log\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}yuanrong_remote_load_hits_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 144, + "gridPos": { + "x": 0, + "y": 52, + "w": 12, + "h": 8 + } + }, + { + "title": "yuanrong_local_ssd_load_hits_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Estimated YuanRong local spill SSD Get hits forwarded from kv_resource.log\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}yuanrong_local_ssd_load_hits_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 145, + "gridPos": { + "x": 12, + "y": 52, + "w": 12, + "h": 8 + } + }, + { + "title": "yuanrong_l2_load_hits_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "YuanRong L2 persistence Get hits forwarded from kv_resource.log\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}yuanrong_l2_load_hits_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 146, + "gridPos": { + "x": 0, + "y": 60, + "w": 12, + "h": 8 + } + }, + { + "title": "yuanrong_resource_log_last_update_timestamp_seconds", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Unix timestamp of the latest YuanRong resource snapshot parsed by UCM\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}yuanrong_resource_log_last_update_timestamp_seconds{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"} * 1000", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "dateTimeFromNow", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 147, + "gridPos": { + "x": 12, + "y": 60, + "w": 12, + "h": 8 + } + }, + { + "title": "yuanrong_resource_log_reporter_leader", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Whether this UCM process is the host YuanRong resource reporter leader\n\nMissing series means not emitted, not zero. Gauge values retain original labels; shared capacity is not summed across workers.", + "targets": [ + { + "expr": "${ucm_prefix}yuanrong_resource_log_reporter_leader{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 148, + "gridPos": { + "x": 0, + "y": 68, + "w": 12, + "h": 8 + } + } + ], + "id": 139, + "gridPos": { + "x": 0, + "y": 35, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Yuanrong / Errors and recovery", + "collapsed": true, + "panels": [ + { + "title": "yuanrong_load_failed_shards_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "YuanRong pipeline load shards that did not complete device delivery\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}yuanrong_load_failed_shards_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 150, + "gridPos": { + "x": 0, + "y": 37, + "w": 12, + "h": 8 + } + }, + { + "title": "yuanrong_resource_log_read_errors_total / rate", + "type": "timeseries", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of failures opening, reading, or parsing YuanRong kv_resource.log\n\nMissing series means not emitted, not zero.", + "targets": [ + { + "expr": "sum by (job, instance, model_name, worker_id) (rate(${ucm_prefix}yuanrong_resource_log_read_errors_total{job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}[$__rate_interval]))", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": false, + "range": true, + "format": "time_series" + } + ], + "fieldConfig": { + "defaults": { + "unit": "ops", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "id": 151, + "gridPos": { + "x": 12, + "y": 37, + "w": 12, + "h": 8 + } + } + ], + "id": 149, + "gridPos": { + "x": 0, + "y": 36, + "w": 24, + "h": 1 + } + }, + { + "type": "row", + "title": "Yuanrong / Cumulative counters and observations", + "collapsed": true, + "panels": [ + { + "title": "Yuanrong / cumulative counter samples", + "type": "table", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Cumulative values since worker start. Histogram count is the number of observations, not a success count.", + "targets": [ + { + "expr": "{__name__=~\"${ucm_prefix}(yuanrong_load_success_shards_total|yuanrong_lookup_miss_posix_load_success_shards_total|yuanrong_load_fallback_posix_load_success_shards_total|yuanrong_load_failed_shards_total|yuanrong_local_dram_load_hits_total|yuanrong_remote_load_hits_total|yuanrong_local_ssd_load_hits_total|yuanrong_l2_load_hits_total|yuanrong_resource_log_read_errors_total)\",job=~\"$job\",instance=~\"$instance\",model_name=~\"$model_name\",worker_id=~\"$worker_id\"}", + "legendFormat": "__auto", + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "instant": true, + "range": false, + "format": "table" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "No data", + "custom": { + "spanNulls": false + } + }, + "overrides": [] + }, + "options": { + "showHeader": true + }, + "id": 153, + "gridPos": { + "x": 0, + "y": 38, + "w": 24, + "h": 8 + } + } + ], + "id": 152, + "gridPos": { + "x": 0, + "y": 37, + "w": 24, + "h": 1 + } + } + ], + "links": [ + { + "type": "link", + "title": "SGLang + UCM / Overview", + "url": "/d/sglang-dashboard", + "includeVars": true, + "keepTime": true, + "targetBlank": false + }, + { + "type": "link", + "title": "SGLang + UCM / HiCache", + "url": "/d/sglang-ucm-hicache", + "includeVars": true, + "keepTime": true, + "targetBlank": false + }, + { + "type": "link", + "title": "SGLang + UCM / Direct Linker", + "url": "/d/sglang-ucm-linker", + "includeVars": true, + "keepTime": true, + "targetBlank": false + } + ] +} diff --git a/test/common/sglang_utils.py b/test/common/sglang_utils.py index 0bb7ec160..9de28f912 100644 --- a/test/common/sglang_utils.py +++ b/test/common/sglang_utils.py @@ -10,8 +10,10 @@ import re import sys import threading +import time import types -from dataclasses import dataclass, replace +from contextlib import contextmanager +from dataclasses import dataclass, field, replace from pathlib import Path from typing import Any, List @@ -44,6 +46,7 @@ def load_source(relative_path, definitions=None, **dependencies): defaults = dict( dataclass=dataclass, replace=replace, + field=field, np=np, copy=copy, hashlib=hashlib, @@ -64,9 +67,20 @@ def load_source(relative_path, definitions=None, **dependencies): return _load_source(relative_path, definitions=definitions, **defaults) -adapter_module = load_source("ucm/integration/sglang/ucm_direct_store_adapter.py") +metrics_module = load_source( + "ucm/integration/sglang/metrics.py", contextmanager=contextmanager, time=time +) +adapter_module = load_source( + "ucm/integration/sglang/ucm_direct_store_adapter.py", + AsyncIoObservation=metrics_module.AsyncIoObservation, + record_metrics=metrics_module.record_metrics, +) linker_module = load_source( "ucm/integration/sglang/ucm_direct_linker.py", + LinkerOperation=metrics_module.LinkerOperation, + record_metrics=metrics_module.record_metrics, + shutdown_metrics_logger=metrics_module.shutdown_metrics_logger, + create_metrics_logger=metrics_module.create_metrics_logger, **{ name: getattr(adapter_module, name) for name in ("UcmDirectStoreAdapter", "UcmLayerTransfer", "UcmPoolMetadata") diff --git a/test/test_sglang_dump_partition.py b/test/test_sglang_dump_partition.py index 0132dfe75..4ad3e0115 100644 --- a/test/test_sglang_dump_partition.py +++ b/test/test_sglang_dump_partition.py @@ -47,6 +47,7 @@ def setUp(self): def linker(self, rank, ranks=(0, 1, 2, 3)): obj = object.__new__(linker_module.UcmDirectLinker) + obj._metrics_enabled = False obj._tp_rank, obj._tp_size = rank, 4 obj._dump_ranks = ranks obj._lookup_owner = True @@ -243,7 +244,8 @@ def test_offload_resolves_once_and_empty_rank_records_no_events(self): expected = obj._partition_offload(transfers) self.assertTrue(obj.offload(transfers)) self.assertEqual(len(expanded), 1) - queued, ready = obj._offload_queue.get_nowait() + queued, ready, observation = obj._offload_queue.get_nowait() + self.assertIsNone(observation) self.assertEqual([t.name for t in queued], [t.name for t in expected]) self.assertEqual(len(events), int(bool(expected))) if not expected: diff --git a/test/test_sglang_linker_metrics.py b/test/test_sglang_linker_metrics.py new file mode 100644 index 000000000..14358253c --- /dev/null +++ b/test/test_sglang_linker_metrics.py @@ -0,0 +1,506 @@ +"""CPU-only tests of actual adapter/worker paths with deterministic metric clocks.""" + +import collections +import json +import queue +import types +import unittest +from pathlib import Path +from unittest.mock import Mock, patch + +import test_sglang_direct_preparation as preparation +import yaml +from common.sglang_utils import ( + adapter_module, + linker_module, + metrics_module, +) + + +class LinkerMetricsTest(unittest.TestCase): + def setUp(self): + self.now = 10.0 + self.samples = [] + for module in (metrics_module, adapter_module, linker_module): + self.enterContext( + patch.object(module, "record_metrics", self.samples.append) + ) + self.enterContext( + patch.object( + metrics_module, + "time", + types.SimpleNamespace(perf_counter=lambda: self.now), + ) + ) + self.enterContext( + patch.object( + linker_module, + "time", + types.SimpleNamespace(perf_counter=lambda: self.now), + create=True, + ) + ) + self.fixture = preparation.LayoutPreparationTest() + self.adapter = self.fixture.adapter() + self.adapter.metrics_enabled = True + self.store = types.SimpleNamespace( + load_data=Mock(side_effect=[1, 2, 3, 4]), + dump_data=Mock(side_effect=[1, 2, 3, 4]), + wait=Mock(), + check=Mock(return_value=True), + ) + self.adapter._ensure_store_schema = lambda *a, **kw: self.store + + def total(self, name): + return sum(sample.get(name, 0) for sample in self.samples) + + def test_prefetched_layers_record_at_wait_with_shard_bytes(self): + prepared = self.adapter.prepare_load([self.fixture.transfer()]) + first = self.adapter.submit_prepared_load_layer(prepared, 0) + self.now = 11 + second = self.adapter.submit_prepared_load_layer(prepared, 2) + self.assertEqual(self.samples, []) + self.now = 13 + self.adapter.wait(second) + self.now = 15 + self.adapter.wait(first) + self.assertEqual(self.total("sglang_connector_load_requests_total"), 2) + self.assertEqual(self.total("sglang_connector_load_bytes_total"), 32) + self.assertEqual(self.total("sglang_connector_load_pages"), 4) + self.assertEqual(self.total("sglang_connector_load_duration_ms"), 7000) + first[0].observation.finish() + self.assertEqual(self.total("sglang_connector_load_requests_total"), 2) + + def test_distributed_load_options_preserve_completion_metrics(self): + self.store.supports_distributed_backend_load = True + for replicated in (False, True): + for event in (0, 43): + with self.subTest(replicated=replicated, event=event): + self.samples.clear() + self.adapter._identity["rank_replicated"] = replicated + prepared = self.adapter.prepare_load([self.fixture.transfer()]) + tasks = self.adapter.submit_prepared_load_layer( + prepared, 0, prerequisite_handle=event + ) + expected = {"distribute_backend_load": replicated} + if event: + expected["prerequisite_handle"] = event + self.assertEqual(self.store.load_data.call_args.kwargs, expected) + self.assertEqual(self.samples, []) + self.adapter.wait(tasks) + self.assertEqual( + self.total("sglang_connector_load_requests_total"), 1 + ) + self.assertEqual( + self.total("sglang_connector_load_bytes_total"), 16 + ) + + def test_distributed_load_submit_failure_drains_and_records_metrics(self): + self.store.supports_distributed_backend_load = True + self.adapter._identity["rank_replicated"] = True + self.store.load_data.side_effect = [1, RuntimeError("submit")] + prepared = self.adapter.prepare_load( + [self.fixture.transfer(), self.fixture.transfer()] + ) + with self.assertLogs(adapter_module.logger, "ERROR"): + with self.assertRaisesRegex(RuntimeError, "submit"): + self.adapter.submit_prepared_load_layer( + prepared, 0, prerequisite_handle=43 + ) + for call in self.store.load_data.call_args_list: + self.assertEqual( + call.kwargs, + {"prerequisite_handle": 43, "distribute_backend_load": True}, + ) + self.store.wait.assert_called_once_with(1) + self.assertEqual(self.total("sglang_connector_load_requests_total"), 2) + self.assertEqual(self.total("sglang_connector_load_submit_errors_total"), 1) + self.assertEqual(self.total("sglang_connector_load_bytes_total"), 16) + + def test_partial_submit_failure_drains_success_and_records_failed_attempt(self): + self.store.dump_data.side_effect = [1, RuntimeError("submit")] + with self.assertLogs(adapter_module.logger, "ERROR"): + with self.assertRaisesRegex(RuntimeError, "submit"): + self.adapter.submit_offload( + [self.fixture.transfer(), self.fixture.transfer()] + ) + self.store.wait.assert_called_once_with(1) + self.assertEqual(self.total("sglang_connector_dump_requests_total"), 2) + self.assertEqual(self.total("sglang_connector_dump_submit_errors_total"), 1) + self.assertEqual(self.total("sglang_connector_dump_bytes_total"), 32) + + def test_wait_failure_does_not_skip_remaining_tasks(self): + tasks = self.adapter.submit_offload( + [self.fixture.transfer(), self.fixture.transfer()] + ) + self.store.wait.side_effect = [RuntimeError("wait"), None] + with self.assertRaisesRegex(RuntimeError, "wait"): + self.adapter.wait(tasks) + self.assertEqual(self.store.wait.call_count, 2) + self.assertEqual(self.total("sglang_connector_dump_wait_errors_total"), 1) + self.assertEqual(self.total("sglang_connector_dump_bytes_total"), 32) + + def test_check_failure_is_not_overwritten_by_successful_cleanup(self): + tasks = self.adapter.submit_offload( + [self.fixture.transfer(), self.fixture.transfer()] + ) + self.store.check.side_effect = RuntimeError("check") + for _ in range(2): + with self.assertRaisesRegex(RuntimeError, "check"): + self.adapter.check(tasks[0]) + self.assertEqual(self.total("sglang_linker_dump_check_errors_total"), 1) + self.assertEqual(self.total("sglang_connector_dump_requests_total"), 0) + self.adapter.wait(tasks) + self.assertEqual(self.total("sglang_connector_dump_wait_errors_total"), 1) + self.assertEqual(self.total("sglang_connector_dump_bytes_total"), 32) + + def test_disabled_and_empty_observations_do_not_read_clock(self): + self.adapter.metrics_enabled = False + with patch.object( + metrics_module.time, "perf_counter", side_effect=AssertionError("clock") + ): + tasks = self.adapter.submit_offload( + [self.fixture.transfer(), self.fixture.transfer()] + ) + self.adapter.wait(tasks) + self.assertIsNone( + metrics_module.AsyncIoObservation.begin(True, False, 0, 8) + ) + self.assertIsNone(metrics_module.LinkerOperation.begin(False, "load")) + self.assertEqual(self.samples, []) + + def test_operation_queue_and_completion_have_separate_boundaries(self): + obs = metrics_module.LinkerOperation.begin(True, "offload") + self.now = 12 + obs.start() + self.assertEqual(self.total("sglang_linker_offload_completed_total"), 0) + self.now = 17 + obs.finish(False) + obs.finish(True) + self.assertEqual(self.total("sglang_linker_offload_queue_duration_ms"), 2000) + self.assertEqual(self.total("sglang_linker_offload_duration_ms"), 7000) + self.assertEqual(self.total("sglang_linker_offload_errors_total"), 1) + self.assertEqual(self.total("sglang_linker_offload_completed_total"), 1) + + def lookup_linker(self): + obj = object.__new__(linker_module.UcmDirectLinker) + obj._metrics_enabled = True + obj._lookup_owner = True + obj._lookup_progress = collections.OrderedDict() + obj._expand = lambda transfers, **kw: transfers + obj._page_transfers = lambda transfers, **kw: [ + self.fixture.transfer(addresses=None) + ] + obj.adapter = types.SimpleNamespace(lookup=Mock(return_value=[2, 4])) + self.enterContext( + patch.object( + linker_module, "PoolName", types.SimpleNamespace(KV="kv"), create=True + ) + ) + return obj, [types.SimpleNamespace(name="kv", keys=["a", "b", "c", "d"])] + + def test_lookup_sparse_boundaries_resume_and_non_owner(self): + obj, transfers = self.lookup_linker() + self.assertEqual(obj.lookup("r", transfers), [2, 4]) + obj.lookup("r", transfers) + self.assertEqual(self.total("sglang_linker_lookup_query_pages_total"), 8) + self.assertEqual(self.total("sglang_linker_lookup_hit_pages_total"), 8) + self.assertEqual(self.total("sglang_linker_lookup_resumed_total"), 1) + obj._lookup_owner = False + self.assertEqual(obj.lookup("r", transfers), [1, 2, 3, 4]) + self.assertEqual(self.total("sglang_linker_lookup_hit_pages_total"), 8) + self.assertEqual(obj.adapter.lookup.call_count, 2) + self.assertEqual(self.total("sglang_linker_lookup_skipped_total"), 1) + + def test_lookup_error_records_no_successful_query_or_hit_pages(self): + obj, transfers = self.lookup_linker() + obj.adapter.lookup.side_effect = RuntimeError("lookup") + with self.assertRaisesRegex(RuntimeError, "lookup"): + obj.lookup("r", transfers) + self.assertEqual(self.total("sglang_linker_lookup_errors_total"), 1) + self.assertEqual(self.total("sglang_linker_lookup_requests_total"), 1) + self.assertEqual(self.total("sglang_linker_lookup_query_pages_total"), 0) + self.assertEqual(len(self.samples), 1) + self.assertNotIn("r", obj._lookup_progress) + + def test_lookup_records_one_batch_without_including_export_time(self): + obj, transfers = self.lookup_linker() + + def lookup(*args, **kwargs): + self.now += 0.25 + return [2, 4] + + def record(values): + self.samples.append(values) + self.now += 1 + + obj.adapter.lookup.side_effect = lookup + with patch.object(linker_module, "record_metrics", record): + self.assertEqual(obj.lookup("r", transfers), [2, 4]) + self.assertEqual(len(self.samples), 1) + self.assertEqual(self.total("sglang_linker_lookup_duration_ms"), 250) + self.assertEqual(self.total("sglang_linker_lookup_query_pages_total"), 4) + self.assertEqual(self.total("sglang_linker_lookup_hit_pages_total"), 4) + self.assertEqual(obj._lookup_progress["r"].hit_pages, 4) + + def test_disabled_lookup_preserves_result_without_clock_or_samples(self): + obj, transfers = self.lookup_linker() + obj._metrics_enabled = False + with patch.object( + linker_module.time, "perf_counter", side_effect=AssertionError("clock") + ): + self.assertEqual(obj.lookup("r", transfers), [2, 4]) + obj.adapter.lookup.side_effect = RuntimeError("lookup") + with self.assertRaisesRegex(RuntimeError, "lookup"): + obj.lookup("failed", transfers) + self.assertEqual(self.samples, []) + self.assertNotIn("failed", obj._lookup_progress) + + def test_resume_fallback_records_actual_probe(self): + self.store.lookup_on_prefix = Mock(side_effect=[-1, 1]) + result = self.adapter.lookup( + [self.fixture.transfer(addresses=None)], + ["a", "b", "c"], + resume_from_pages=3, + ) + self.assertEqual(result, [1, 2]) + self.assertEqual(self.total("sglang_linker_lookup_fallback_total"), 1) + + def test_load_worker_counts_failure_and_drains_prefetched_tasks(self): + obj = object.__new__(linker_module.UcmDirectLinker) + obj._metrics_enabled, obj._tp_rank = True, 0 + obj._requires_load_finish = False + obj._load_queue, obj._completed_loads = queue.Queue(), queue.Queue() + obj._page_transfers = lambda *a, **kw: [self.fixture.transfer()] + obj.num_layers = 3 + self.store.check.return_value = False + obj.layer_done_counter = types.SimpleNamespace(complete=Mock(), fail=Mock()) + obj.adapter = self.adapter + self.store.wait.side_effect = [RuntimeError("load"), None] + observation = metrics_module.LinkerOperation.begin(True, "load") + obj._load_queue.put( + ( + 1, + {"rid": []}, + [types.SimpleNamespace(synchronize=lambda: None)], + observation, + ) + ) + obj._load_queue.put(None) + with self.assertLogs(linker_module.logger, "ERROR"): + obj._load_worker() + self.assertEqual(self.store.wait.call_count, 2) + self.assertEqual(self.total("sglang_linker_load_errors_total"), 1) + self.assertEqual(self.total("sglang_linker_load_completed_total"), 1) + self.assertEqual(self.total("sglang_connector_load_bytes_total"), 16) + self.assertEqual(obj._load_queue.unfinished_tasks, 0) + + def test_config_overrides_and_disabled_default_adapter(self): + document = { + "enable_metrics": True, + "metrics_config_path": "old.yaml", + "ucm_connectors": [ + { + "ucm_connector_name": "test", + "ucm_connector_config": {}, + } + ], + } + self.enterContext(patch.object(adapter_module, "yaml", yaml, create=True)) + self.enterContext( + patch.object( + Path, + "open", + unittest.mock.mock_open(read_data=yaml.safe_dump(document)), + ) + ) + adapter = adapter_module.UcmDirectStoreAdapter.from_config( + { + "UCM_CONFIG_FILE": "config.yaml", + "enable_metrics": False, + "metrics_config_path": "new.yaml", + }, + {}, + ) + self.assertEqual( + adapter.launch_config, + {"enable_metrics": False, "metrics_config_path": "new.yaml"}, + ) + self.assertFalse(adapter.metrics_enabled) + self.assertNotIn("enable_metrics", adapter._store_config) + + def test_linker_logger_identity_config_and_disable(self): + obj = object.__new__(linker_module.UcmDirectLinker) + obj._dp_rank, obj._attn_dp_rank, obj._tp_rank = 0, 3, 1 + obj.adapter = types.SimpleNamespace( + launch_config={"metrics_config_path": "file.yaml"} + ) + params = types.SimpleNamespace(enable_metrics=True, pp_rank=2, attn_cp_rank=0) + self.enterContext( + patch.object( + linker_module, + "get_model", + lambda: types.SimpleNamespace(model_path="model"), + create=True, + ) + ) + self.enterContext( + patch.object( + linker_module, + "get_parallel", + lambda: types.SimpleNamespace(enable_dp_attention=True), + create=True, + ) + ) + factory = self.enterContext( + patch.object( + linker_module, + "create_metrics_logger", + return_value=types.SimpleNamespace(config={"counter": []}), + ) + ) + obj._init_metrics(params, {"metrics_config_path": "override.yaml"}) + self.assertTrue(obj.adapter.metrics_enabled) + self.assertEqual(factory.call_args.kwargs["ranks"]["dp_rank"], 3) + self.assertEqual( + factory.call_args.kwargs["launch_config"]["metrics_config_path"], + "override.yaml", + ) + factory.return_value = None + params.enable_metrics = False + obj._init_metrics(params, {}) + self.assertFalse(obj.adapter.metrics_enabled) + + def test_shutdown_stops_poller_before_final_export(self): + events = [] + dispatcher = types.SimpleNamespace( + drain_to_consumers=lambda: events.append("drain"), + get_stats_and_clear=lambda name: ({"last": 1}, {}, {}), + ) + logger = types.SimpleNamespace( + shutdown=lambda: events.append("stop"), + metrics_dispatcher=dispatcher, + update_stats=lambda *stats: events.append(stats[0]), + ) + metrics_module.shutdown_metrics_logger(logger) + self.assertEqual(events, ["stop", "drain", {"last": 1}]) + + def test_registration_precedes_store_setup_and_failure_closes_logger(self): + events = [] + group = types.SimpleNamespace(entry_map={}, num_layers=0, rank_replicated=False) + parallel = types.SimpleNamespace(tp_size=1, dp_rank=0, attn_dp_rank=0) + params = types.SimpleNamespace( + page_size=4, + token_to_kv_pool_allocator=types.SimpleNamespace( + get_kvcache=lambda: object() + ), + attn_tp_cache_group=None, + tp_cache_group=None, + attn_cp_size=1, + attn_cp_rank=0, + pp_rank=0, + enable_metrics=True, + ) + logger = types.SimpleNamespace(config={"counter": []}) + + def create(**kwargs): + events.append("logger") + return logger + + def register(pools): + events.append("store") + raise RuntimeError("Store setup failed") + + adapter = types.SimpleNamespace( + launch_config={}, + register_device_pools=register, + close=lambda: events.append("store.close"), + ) + for name, value in { + "get_parallel": lambda: parallel, + "get_model": lambda: types.SimpleNamespace(model_path="model"), + "resolve_hybrid_device_pool_group": lambda **kwargs: group, + "torch": types.SimpleNamespace( + distributed=types.SimpleNamespace(is_available=lambda: False) + ), + "Queue": queue.Queue, + "OrderedDict": collections.OrderedDict, + "create_metrics_logger": create, + "shutdown_metrics_logger": lambda instance: events.append("logger.close"), + }.items(): + self.enterContext(patch.object(linker_module, name, value, create=True)) + cls = linker_module.UcmDirectLinker + self.enterContext(patch.object(cls, "_build_pool_bundles", return_value=())) + self.enterContext( + patch.object(cls, "_build_layer_pointer_layouts", return_value={}) + ) + self.enterContext(patch.object(cls, "_log_pool_topology")) + with self.assertLogs(linker_module.logger, "ERROR"): + with self.assertRaisesRegex(RuntimeError, "Store setup failed"): + cls(None, params, components=[], adapter=adapter) + self.assertEqual(events, ["logger", "store", "store.close", "logger.close"]) + + def test_non_owner_offload_and_pending_cancel_never_create_io_samples(self): + obj = object.__new__(linker_module.UcmDirectLinker) + obj._metrics_enabled = True + obj._partition_offload = lambda transfers: [] + obj._offload_queue = queue.Queue() + obj._offload_results = queue.Queue() + obj._expand = lambda transfers, **kwargs: transfers + obj._freeze_gc_once = lambda: None + obj._pending_loads, obj._lookup_progress = {}, {} + self.assertTrue(obj.offload([object()])) + self.assertEqual(obj._offload_queue.get_nowait(), ([], [], None)) + self.assertEqual(obj.num_completed_offloads(), 0) + self.assertTrue(obj.load("request", [object()])) + self.assertTrue(obj.cancel_queued_load("request")) + self.assertFalse(obj.cancel_queued_load("request")) + self.assertEqual(self.total("sglang_linker_offload_skipped_total"), 1) + self.assertEqual(self.total("sglang_linker_load_cancelled_requests_total"), 1) + self.assertFalse( + any( + key.startswith("sglang_connector_") + for sample in self.samples + for key in sample + ) + ) + + +class DashboardTest(unittest.TestCase): + def test_dashboard_metric_names_and_panel_ids(self): + root = Path(__file__).resolve().parents[1] / "examples/metrics" + config = yaml.safe_load( + (root / "metrics_configs_sglang.yaml").read_text(encoding="utf-8") + ) + names = { + item["name"] + for kind in ("counter", "gauge", "histogram") + for item in config[kind] + } + for item in config["histogram"]: + names.update( + item["name"] + suffix for suffix in ("_sum", "_count", "_bucket") + ) + board = json.loads( + (root / "sglang-linker-dashboard.json").read_text(encoding="utf-8") + ) + ids = [panel["id"] for panel in board["panels"]] + self.assertEqual(len(ids), len(set(ids))) + import re + + queried = set() + for panel in board["panels"]: + for target in panel.get("targets", []): + expr = target["expr"] + self.assertNotIn("or vector(0)", expr) + self.assertIn('worker_id=~"$worker_id"', expr) + for name in re.findall(r"\$\{ucm_prefix\}(sglang_\w+)\{", expr): + self.assertIn(name, names) + queried.add(name) + for item in config["counter"]: + if item["name"].startswith("sglang_linker_"): + self.assertIn(item["name"], queried) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_sglang_load_workers.py b/test/test_sglang_load_workers.py index df52e3e77..8931b1b30 100644 --- a/test/test_sglang_load_workers.py +++ b/test/test_sglang_load_workers.py @@ -23,6 +23,7 @@ def run_worker( submitted = [] reaped = set() obj = object.__new__(linker_module.UcmDirectLinker) + obj._metrics_enabled = False obj.num_layers = 4 obj._tp_rank = 0 obj._requires_load_finish = True @@ -87,7 +88,7 @@ def wait(tasks): synchronize=lambda: events.append(("forward_done", None)), ) ) - obj._load_queue.put((0, {"req": [object()]}, ready_events)) + obj._load_queue.put((0, {"req": [object()]}, ready_events, None)) obj._load_queue.put(None) obj._load_worker() self.assertEqual(obj._load_queue.unfinished_tasks, 0) @@ -142,6 +143,7 @@ def event(): ) obj = object.__new__(linker_module.UcmDirectLinker) + obj._metrics_enabled = False obj._pending_loads = {"req": [object()]} obj.forward_stream = forward_stream obj._requires_load_finish = False @@ -157,6 +159,7 @@ def event(): def test_native_transfer_ready_is_reused_for_load_and_dump(self): obj = object.__new__(linker_module.UcmDirectLinker) + obj._metrics_enabled = False ready_events = [object(), object()] obj.forward_stream = object() obj._pending_loads = {"req": [object()]} @@ -188,6 +191,7 @@ def record_ready(linker): def test_legacy_transfer_ready_without_forward_stream(self): obj = object.__new__(linker_module.UcmDirectLinker) + obj._metrics_enabled = False obj.forward_stream = None recorded = [] event = types.SimpleNamespace(record=lambda: recorded.append(True)) diff --git a/test/test_sglang_metrics.py b/test/test_sglang_metrics.py new file mode 100644 index 000000000..4ad67aec0 --- /dev/null +++ b/test/test_sglang_metrics.py @@ -0,0 +1,555 @@ +"""Test real multiprocess Prometheus export without native UCM, SGLang or GPUs. + +Run with pytest --noconftest test/test_sglang_metrics.py. Each case uses a fresh +interpreter because Prometheus selects its storage mode at import time. +""" + +import os +import subprocess +import sys +import textwrap +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +BOOTSTRAP = r""" +import logging, os, sys, time, types +from pathlib import Path +import yaml + +def module(name, **attrs): + result = types.ModuleType(name) + result.__dict__.update(attrs) + sys.modules[name] = result + +module("ucm", __path__=[str(Path.cwd() / "ucm")]) +module("ucm.shared", __path__=[str(Path.cwd() / "ucm/shared")]) +module("ucm.logger", init_logger=logging.getLogger) +snapshot = ({}, {}, {}) +registered, events = [], [] +drains = 0 +def drain(): + global snapshot, drains + result, snapshot = snapshot, ({}, {}, {}) + drains += 1 + return result +module("ucm.shared.metrics", ucmmetrics=types.SimpleNamespace( + set_up=lambda: None, create_stats=lambda *a: registered.append(a), + get_all_stats_and_clear=drain)) +module("torch") +for name in ("sglang", "sglang.srt", "sglang.srt.distributed"): + module(name, __path__=[]) +module("sglang.srt.distributed.parallel_state", + get_world_group=lambda: types.SimpleNamespace(local_rank=0)) +class Store: + def close(self): + events.append("store.close") +def create(*args): + events.append("store.create") + return Store() +module("ucm.store.factory_v1", UcmConnectorFactoryV1=types.SimpleNamespace(create_connector=create)) +from ucm.integration.sglang import ucm_connector as c + +config = {"log_interval": 0.02, + "consumers": {"multiproc": True, "vllm_connector": False}, + "counter": [{"name": "posix_s2h_bytes_total"}], + "gauge": [{"name": "posix_store_used_bytes", "multiprocess_mode": "livemostrecent"}], + "histogram": [{"name": "posix_load_task_duration_ms", "buckets": [1, 10]}]} +config_path = Path(sys.argv[1]) / "metrics.yaml" +def write_config(): + config_path.write_text(yaml.safe_dump(config), encoding="utf-8") +write_config() +launch = {"metrics_config_path": str(config_path), "ucm_connector_name": "FakeStore", + "ucm_connector_config": {"storage_backends": "/tmp/fake-store"}} +storage = types.SimpleNamespace(model_name="test-model", is_mla_model=False, + tp_rank=0, tp_size=2, pp_rank=0, pp_size=1, dp_rank=0, attn_cp_rank=0, + enable_storage_metrics=True, extra_config={"kv_connector_extra_config": launch}) +pool = types.SimpleNamespace(dtype="f16", page_size=1, kv_buffer=object(), + get_size_per_token=lambda: 128) +def start(): + return c.SglangUcmConnector.from_hicache(storage, pool) +def scrape(): + from prometheus_client import CollectorRegistry, generate_latest, multiprocess + registry = CollectorRegistry() + multiprocess.MultiProcessCollector(registry) + return generate_latest(registry).decode() +def value(name, **labels): + from prometheus_client.parser import text_string_to_metric_families + matches = [s.value for f in text_string_to_metric_families(scrape()) for s in f.samples + if s.name == name and all(s.labels.get(k) == v for k, v in labels.items())] + assert len(matches) == 1, (name, labels, matches) + return matches[0] +""" + + +def run_child(tmp_path, code, *, multiprocess=True): + env = os.environ.copy() + env.pop("prometheus_multiproc_dir", None) + env.pop("PROMETHEUS_MULTIPROC_DIR", None) + if multiprocess: + env["PROMETHEUS_MULTIPROC_DIR"] = str(tmp_path) + result = subprocess.run( + [sys.executable, "-c", BOOTSTRAP + "\n" + textwrap.dedent(code), str(tmp_path)], + cwd=ROOT, + env=env, + capture_output=True, + text=True, + timeout=20, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_two_workers_existing_thread_and_asgi_endpoint(tmp_path): + for rank, amount in [(0, 4096), (1, 8192)]: + run_child( + tmp_path, + f""" + storage.tp_rank = {rank} + def create_with_metrics(*args): + global snapshot + assert registered, "Register metrics before creating stores" + snapshot = ({{"posix_s2h_bytes_total": {amount}}}, + {{"posix_store_used_bytes": {amount}}}, + {{"posix_load_task_duration_ms": ([1, 1, 0], 6.0)}}) + return Store() + c.UcmConnectorFactoryV1.create_connector = create_with_metrics + connector = start() + try: + from prometheus_client import Counter + Counter("sglang:generated_tokens_total", "Generated tokens").inc(3) + deadline = time.monotonic() + 5 + while (snapshot[0] or drains < 3) and time.monotonic() < deadline: + time.sleep(0.01) + assert not snapshot[0] and drains >= 3 + assert value("ucm:posix_s2h_bytes_total", worker_id="dp0_tp{rank}_pp0_attn_cp0") == {amount} + try: + start() + except RuntimeError as error: + assert "only one" in str(error) + else: + raise AssertionError("Duplicate logger accepted") + finally: + connector.close() + assert not connector._metrics_logger.thread.is_alive() + """, + ) + assert len(list(tmp_path.glob("counter_*.db"))) == 2 + run_child( + tmp_path, + """ + import asyncio + from prometheus_client import CollectorRegistry, make_asgi_app, multiprocess + registry = CollectorRegistry() + multiprocess.MultiProcessCollector(registry) + responses = [] + async def receive(): + return {"type": "http.request", "body": b"", "more_body": False} + async def send(message): + responses.append(message) + asyncio.run(make_asgi_app(registry=registry)( + {"type": "http", "method": "GET", "path": "/metrics", "headers": [], + "query_string": b""}, receive, send)) + assert responses[0]["status"] == 200 + text = b"".join(r.get("body", b"") for r in responses).decode() + assert "sglang:generated_tokens_total 6.0" in text + assert "ucm:posix_s2h_bytes_total" in text + worker = "dp0_tp1_pp0_attn_cp0" + assert value("ucm:posix_s2h_bytes_total", worker_id=worker) == 8192 + assert value("ucm:posix_store_used_bytes", worker_id=worker) == 8192 + assert value("ucm:posix_load_task_duration_ms_count", worker_id=worker) == 2 + assert value("ucm:posix_load_task_duration_ms_sum", worker_id=worker) == 6 + assert value("ucm:posix_load_task_duration_ms_bucket", worker_id=worker, le="10.0") == 2 + assert scrape() == scrape() + """, + ) + + +@pytest.mark.parametrize("disable", ["sglang", "ucm"]) +def test_disabled_does_not_import_prometheus(tmp_path, disable): + run_child( + tmp_path, + f""" + if {disable!r} == "sglang": + storage.enable_storage_metrics = False + else: + launch["enable_metrics"] = False + launch.pop("metrics_config_path") + connector = start() + assert connector._metrics_logger is None + assert "prometheus_client" not in sys.modules + connector.close() + """, + multiprocess=False, + ) + + +@pytest.mark.parametrize("early_import", [False, True]) +def test_multiprocess_environment_errors(tmp_path, early_import): + code = "" + if early_import: + code = f'from prometheus_client import Counter\nos.environ["PROMETHEUS_MULTIPROC_DIR"] = {str(tmp_path)!r}\n' + run_child( + tmp_path, + code + + """ +try: + start() +except RuntimeError as error: + assert "PROMETHEUS_MULTIPROC_DIR" in str(error) +else: + raise AssertionError("Invalid multiprocess environment accepted") +assert events == [] +""", + multiprocess=False, + ) + + +def test_vllm_only_yaml_rejected(tmp_path): + run_child( + tmp_path, + """ + config["consumers"] = {"vllm_connector": True} + write_config() + try: + start() + except ValueError as error: + assert "enable multiproc" in str(error) + else: + raise AssertionError("Wrong consumer config accepted") + assert events == [] + """, + ) + + +@pytest.mark.parametrize("contents", [None, "{}", "counter: ["]) +def test_invalid_explicit_config_does_not_fall_back(tmp_path, contents): + run_child( + tmp_path, + f""" + config_path.unlink() + if {contents!r} is not None: + config_path.write_text({contents!r}, encoding="utf-8") + try: + start() + except ValueError as error: + assert "No UCM metrics defined" in str(error) + else: + raise AssertionError("Invalid explicit config silently replaced by defaults") + assert events == [] + """, + ) + + +def test_lifecycle_creation_failure_hybrid_pools_and_close_failure(tmp_path): + run_child( + tmp_path, + """ + class Logger: + def shutdown(self): + events.append("shutdown") + def create_logger(*args): + events.append("logger.create") + return Logger() + c._create_metrics_logger = create_logger + connector = start() + assert events == ["logger.create", "store.create"] + connector.pool_components = {"mamba": [(Store(), 1)], "swa": [(Store(), 1)]} + connector.close() + connector.close() + assert events == ["logger.create", "store.create", "store.close", + "store.close", "store.close", "shutdown"] + events.clear() + def fail(*args): + raise ValueError("store creation failed") + c.UcmConnectorFactoryV1.create_connector = fail + try: + start() + except ValueError: + pass + else: + raise AssertionError("Expected store creation failure") + assert events == ["logger.create", "shutdown"] + events.clear() + pool.kv_buffer = None + connector = start() + assert events == ["logger.create"] + class BrokenStore: + def close(self): + events.append("broken.close") + raise RuntimeError("close failed") + connector.pool_components = {"pool": [(BrokenStore(), 1), (Store(), 1)]} + try: + connector.close() + except RuntimeError: + pass + else: + raise AssertionError("Expected store close failure") + assert events == ["logger.create", "broken.close", "store.close", "shutdown"] + """, + ) + + +IO_BOOTSTRAP = r""" +import bisect, threading +from ucm.integration.sglang import metrics as io_metrics +from ucm.shared.metrics import ucmmetrics + +config = yaml.safe_load(Path("examples/metrics/metrics_configs_sglang.yaml").read_text(encoding="utf-8")) +assert config["consumers"] == {"multiproc": True, "vllm_connector": False} +config["log_interval"] = 0.01 +write_config() +lock = threading.Lock() +observations = [] +defs = {m["name"]: (typ, m) for typ in ("counter", "gauge", "histogram") for m in config[typ]} +def update(values): + with lock: + observations.append(dict(values)) + for name, v in values.items(): + typ, definition = defs[name] + if typ == "counter": + snapshot[0][name] = snapshot[0].get(name, 0) + v + elif typ == "histogram": + bounds = definition["buckets"] + [float("inf")] + counts, total = snapshot[2].get(name, ([0] * len(bounds), 0)) + counts[bisect.bisect_left(bounds, v)] += 1 + snapshot[2][name] = (counts, total + v) +def io_drain(): + with lock: + return drain() +ucmmetrics.update_stats = update +ucmmetrics.get_all_stats_and_clear = io_drain + +class IoStore(Store): + phase = None + def load_data(self, *args): + if self.phase == "submit": + raise RuntimeError("submit failed") + return object() + dump_data = load_data + def wait(self, task): + if self.phase == "wait": + raise RuntimeError("wait failed") +store = IoStore() +def make_connector(): + c.UcmConnectorFactoryV1.create_connector = lambda *args: store + connector = start() + connector._generate_task = lambda keys, indices: (keys, [0] * len(keys), [[1, 2]] * len(keys)) + return connector +def clock_calls(count): + # Exactly 10ms per call, independent of the periodic export thread. + ticks = iter(v for i in range(count) for v in (i, i + 0.01)) + io_metrics.time = types.SimpleNamespace(perf_counter=lambda: next(ticks)) +""" + + +def test_builtin_config_exports_without_yaml_and_preserves_vllm(tmp_path): + run_child( + tmp_path, + IO_BOOTSTRAP + + r""" +from copy import deepcopy +from ucm.default_metrics_config import DEFAULT_METRICS_CONFIG, get_default_metrics_config +from ucm.integration.sglang.default_metrics_config import get_default_sglang_metrics_config +import ucm.metrics_config as metrics_config + +baseline = deepcopy(DEFAULT_METRICS_CONFIG) +expected = yaml.safe_load(Path("examples/metrics/metrics_configs_sglang.yaml").read_text(encoding="utf-8")) +assert get_default_sglang_metrics_config() == expected +modified = get_default_sglang_metrics_config() +modified["counter"][0]["name"] = "changed" +modified["histogram"][-1]["buckets"].append(999999) +modified["consumers"]["multiproc"] = False +assert get_default_sglang_metrics_config() == expected +assert DEFAULT_METRICS_CONFIG == baseline == get_default_metrics_config() + +launch.pop("metrics_config_path") +config_path.unlink() +def reject_yaml(*args, **kwargs): + raise AssertionError("Built-in metrics must not read YAML") +metrics_config.load_metrics_config = reject_yaml +yaml.safe_load = reject_yaml +connector = make_connector() +try: + assert connector._metrics_logger.config == expected + clock_calls(1) + assert connector.batch_get_v1(["a"], None) == [True] + deadline = time.monotonic() + 7 + while time.monotonic() < deadline: + try: + if value("ucm:sglang_connector_load_bytes_total") == 128: + break + except AssertionError: + pass + time.sleep(0.02) + assert value("ucm:sglang_connector_load_requests_total") == 1 + assert value("ucm:sglang_connector_load_bytes_total") == 128 + assert value("ucm:sglang_connector_load_pages_count") == 1 + assert DEFAULT_METRICS_CONFIG == baseline +finally: + connector.close() +""", + ) + + +def test_linker_async_adapter_exports_through_shared_multiprocess_registry(tmp_path): + run_child( + tmp_path, + IO_BOOTSTRAP + + r""" +import numpy as np +module("ucm.store.ucmstore_v1", Task=object, UcmKVStoreBaseV1=object) +from ucm.integration.sglang.ucm_direct_store_adapter import UcmDirectStoreAdapter, UcmLayerTransfer + +metrics_logger = io_metrics.create_metrics_logger( + enabled=True, model_name="linker-model", + ranks={"dp_rank": 1, "tp_rank": 2, "pp_rank": 3, "attn_cp_rank": 0}, + launch_config=launch) +adapter = UcmDirectStoreAdapter(connector_name="test", module_path=None, store_config={}, identity={}) +adapter.metrics_enabled = True +adapter._ensure_store_schema = lambda *a, **kw: store +transfer = UcmLayerTransfer( + store_group="kv", source_pool="kv", pools=("kv",), layers=(0, 1), + shard_size=8, tensor_size_list=(8,), page_keys=("a", "b"), + addresses=np.array([[[100],[108]],[[200],[208]]], dtype=np.uint64), + fragments=((8,0),(8,0))) +operation = io_metrics.LinkerOperation.begin(True, "offload") +operation.start() +tasks = adapter.submit_offload([transfer]) +assert not any("sglang_connector_dump_requests_total" in s for s in observations) +adapter.wait(tasks) +operation.finish(True) +io_metrics.shutdown_metrics_logger(metrics_logger) +assert not metrics_logger.thread.is_alive() +labels = {"model_name": "linker-model", "worker_id": "dp1_tp2_pp3_attn_cp0"} +assert value("ucm:sglang_connector_dump_requests_total", **labels) == 1 +assert value("ucm:sglang_connector_dump_bytes_total", **labels) == 32 +assert value("ucm:sglang_connector_dump_duration_ms_count", **labels) == 1 +assert value("ucm:sglang_linker_offload_completed_total", **labels) == 1 +assert value("ucm:sglang_linker_offload_duration_ms_count", **labels) == 1 +assert "sglang:prefetch_pgs" not in scrape() +""", + ) + + +def test_io_samples_export_across_tp_pp_workers(tmp_path): + for tp, pp in ((0, 0), (1, 1)): + run_child( + tmp_path, + IO_BOOTSTRAP + + f""" +storage.tp_rank, storage.pp_rank, storage.pp_size = {tp}, {pp}, 2 +# Keep PP-scoped test storage paths inside the pytest directory. +launch["ucm_connector_config"]["storage_backends"] = str(Path(sys.argv[1]) / "store") +connector = make_connector() +try: + clock_calls(3) + assert connector.batch_get_v1(["a", "b"], None) == [True, True] + assert connector.batch_get_v1(["c"], None) == [True] + assert connector.batch_set_v1(["a", "b"], None) == [True, True] + assert len(observations) == 3 + worker = "dp0_tp{tp}_pp{pp}_attn_cp0" + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + try: + if value("ucm:sglang_connector_dump_bytes_total", worker_id=worker) == 256: + break + except AssertionError: + pass + time.sleep(0.01) + assert value("ucm:sglang_connector_load_requests_total", worker_id=worker) == 2 + assert value("ucm:sglang_connector_load_pages_count", worker_id=worker) == 2 + assert value("ucm:sglang_connector_load_pages_sum", worker_id=worker) == 3 + assert value("ucm:sglang_connector_load_bytes_total", worker_id=worker) == 384 + assert abs(value("ucm:sglang_connector_load_duration_ms_sum", worker_id=worker) - 20) < 1e-6 + assert abs(value("ucm:sglang_connector_load_bandwidth_gbps_sum", worker_id=worker) - 384/0.01/1e9) < 1e-12 + assert value("ucm:sglang_connector_dump_bytes_total", worker_id=worker) == 256 + # Repeated collection must not replay the same observations. + time.sleep(0.03) + assert value("ucm:sglang_connector_load_requests_total", worker_id=worker) == 2 +finally: + connector.close() +""", + ) + + +def test_io_failures_disabled_empty_and_logical_calls(tmp_path): + run_child( + tmp_path, + IO_BOOTSTRAP + + r""" +connector = make_connector() +try: + clock_calls(4) + for is_set in (False, True): + call = connector.batch_set_v1 if is_set else connector.batch_get_v1 + prefix = "sglang_connector_dump" if is_set else "sglang_connector_load" + store.phase = "submit" + try: + call(["a"], None) + except RuntimeError as exc: + assert str(exc) == "submit failed" + else: + raise AssertionError("submission exception swallowed") + assert observations[-1][prefix + "_submit_errors_total"] == 1 + store.phase = "wait" + assert call(["a"], None) == [False] + assert observations[-1][prefix + "_wait_errors_total"] == 1 + assert all(not any(k.endswith(("_bytes_total", "_bandwidth_gbps", "_pages")) for k in v) for v in observations) + assert connector.batch_get_v1([], None) == [] + pool.kv_buffer = None + connector.store = None + assert connector.batch_get_v1(["a"], None) == [True] + assert connector.batch_set_v1(["a"], None) == [True] + pool.kv_buffer = object() + connector.store = store + store.phase = None + logger = connector._metrics_logger + connector._metrics_logger = None + assert connector.batch_get_v1(["a"], None) == [True] + connector._metrics_logger = logger + assert len(observations) == 4 # No extra clock read or sample for skipped/disabled calls. +finally: + connector.close() +""", + ) + + +def test_io_v2_pool_sizes_flattened_and_recording_failure(tmp_path): + run_child( + tmp_path, + IO_BOOTSTRAP + + r""" +connector = make_connector() +try: + clock_calls(5) + transfer = types.SimpleNamespace(name="swa", keys=["a", "b"]) + connector.pool_components["swa"] = [(store, 48)] + connector._transfer_meta = lambda t: (t.keys, [[1, 2], [3, 4]], [[48, 48], [48, 48]]) + assert connector.batch_io_v2([transfer], False) == {"swa": [True, True]} + assert observations[-1]["sglang_connector_load_bytes_total"] == 192 + store.phase = "submit" + assert connector.batch_io_v2([transfer], True) == {"swa": [False, False]} + assert observations[-1]["sglang_connector_dump_submit_errors_total"] == 1 + store.phase = None + page = types.SimpleNamespace(numel=lambda: 80, element_size=lambda: 1, data_ptr=lambda: 7) + restored = [] + connector.registered_pools["mamba"] = types.SimpleNamespace( + page_size=1, get_data_page=lambda *a, **k: page, + get_dummy_flat_data_page=lambda: page, + set_from_flat_data_page=lambda *a: restored.append(a)) + connector.pool_components["mamba"] = [(store, 80)] + connector.flattened_pools.add("mamba") + flat = types.SimpleNamespace(name="mamba", keys=["a"], host_indices=[types.SimpleNamespace(item=lambda: 0)]) + assert connector.batch_io_v2([flat], True) == {"mamba": [True]} + assert observations[-1]["sglang_connector_dump_bytes_total"] == 80 + assert connector.batch_io_v2([flat], False) == {"mamba": [True]} + assert len(restored) == 1 + assert observations[-1]["sglang_connector_load_bytes_total"] == 80 + # A telemetry failure must not fail a successful storage operation. + ucmmetrics.update_stats = lambda values: (_ for _ in ()).throw(RuntimeError("metrics unavailable")) + assert connector.batch_get_v1(["a"], None) == [True] +finally: + connector.close() +""", + ) diff --git a/test/test_sglang_offload_workers.py b/test/test_sglang_offload_workers.py index 05bf11182..260e02377 100644 --- a/test/test_sglang_offload_workers.py +++ b/test/test_sglang_offload_workers.py @@ -6,8 +6,9 @@ import time import types import unittest +from unittest.mock import patch -from common.sglang_utils import linker_module +from common.sglang_utils import linker_module, metrics_module linker_module.deque = collections.deque linker_module.Empty = queue.Empty @@ -25,6 +26,7 @@ def setUp(self): self.events = {} obj = object.__new__(linker_module.UcmDirectLinker) obj._tp_rank, obj._closed = 0, False + obj._metrics_enabled, obj._metrics_logger = False, None obj._offload_queue, obj._offload_results = queue.Queue(), queue.Queue() obj._load_queue, obj._completed_loads = queue.Queue(), queue.Queue() obj._load_results, obj._lookup_progress, obj._pending_loads = {}, {}, {} @@ -85,7 +87,45 @@ def enqueue(self, key): cuda_event=77, synchronize=lambda: self.fail("KV event should be forwarded to Store"), ) - self.obj._offload_queue.put(([key], [index_event, kv_event])) + observation = metrics_module.LinkerOperation.begin( + self.obj._metrics_enabled, "offload" + ) + self.obj._offload_queue.put(([key], [index_event, kv_event], observation)) + + def test_metrics_complete_at_reap_before_ordered_publication(self): + samples = [] + completed = threading.Event() + + def record(values): + samples.append(values) + if values.get("sglang_linker_offload_completed_total"): + completed.set() + + self.enterContext(patch.object(metrics_module, "record_metrics", record)) + self.enterContext(patch.object(linker_module, "record_metrics", samples.append)) + self.obj._metrics_enabled = True + self.enqueue("A") + self.enqueue("B") + self.complete["B"].set() + self.assertTrue(self.waited["B"].wait(2)) + # A blocks ordered publication, while B must already be observed. + self.assertTrue(completed.wait(2)) + self.assertTrue(self.obj._offload_results.empty()) + self.assertEqual( + sum(s.get("sglang_linker_offload_completed_total", 0) for s in samples), 1 + ) + self.complete["A"].set() + self.assertTrue(self.obj._offload_results.get(timeout=2)) + self.assertTrue(self.obj._offload_results.get(timeout=2)) + self.assertEqual( + sum(s.get("sglang_linker_offload_completed_total", 0) for s in samples), 2 + ) + self.assertEqual( + sum(s.get("sglang_linker_offload_operations_total", 0) for s in samples), 2 + ) + self.assertEqual( + sum(s.get("sglang_linker_offload_errors_total", 0) for s in samples), 0 + ) def test_pending_tasks_do_not_block_new_submissions(self): self.enqueue("A") @@ -130,7 +170,7 @@ def test_wait_failure_is_published_and_next_node_completes(self): def test_empty_rank_result_does_not_overtake_owned_node(self): self.enqueue("A") - self.obj._offload_queue.put(([], [])) + self.obj._offload_queue.put(([], [], None)) self.enqueue("C") self.complete["C"].set() self.assertTrue(self.waited["C"].wait(2)) diff --git a/ucm/integration/sglang/default_metrics_config.py b/ucm/integration/sglang/default_metrics_config.py new file mode 100644 index 000000000..b65037392 --- /dev/null +++ b/ucm/integration/sglang/default_metrics_config.py @@ -0,0 +1,165 @@ +"""Built-in SGLang metrics for HiCache and direct Linker adapters. + +Common definitions are copied from UCM defaults; vLLM configuration is never mutated. +Keep these additions in sync with examples/metrics/metrics_configs_sglang.yaml. +""" + +from ucm.default_metrics_config import get_default_metrics_config + + +def get_linker_metric_definitions() -> dict: + """Bounded per-worker series; no request IDs, layers or pool IDs as labels.""" + counters = { + "lookup_requests_total": "Owner lookup attempts; excludes neutral non-owner responses", + "lookup_errors_total": "Failed owner lookup attempts", + "lookup_query_pages_total": "Full requested KV pages on successful owner lookups, including resumed prefixes; repeated lookups count again", + "lookup_hit_pages_total": "Maximum locally restorable KV page boundary on successful owner lookups; before cross-rank intersection, not final token reuse", + "lookup_resumed_total": "Owner lookup attempts with a previous hit boundary", + "lookup_fallback_total": "Per-group prefix probes restarted at page zero after the cached boundary disappeared", + "lookup_skipped_total": "Neutral lookup responses on non-owner ranks; not actual storage hits", + "load_requests_total": "Requests accepted into the scheduler's pending load set", + "load_cancelled_requests_total": "Pending requests removed by cancellation or reset before batching", + "load_ready_layers_total": "Logical layers published ready across load batches; not physical Store tasks", + "offload_skipped_total": "Offload calls acknowledged on non-owner ranks without storage I/O", + "load_check_errors_total": "Physical load tasks with a Check failure; also included in connector wait errors when reaped", + "dump_check_errors_total": "Physical dump tasks with a Check failure; also included in connector wait errors when reaped", + } + histograms = [ + { + "name": "sglang_linker_lookup_duration_ms", + "documentation": "SGLang linker: Owner adapter lookup duration including failed attempts (ms)", + "buckets": [0.01, 0.1, 0.5, 1, 5, 10, 50, 100, 500, 1000, 5000, 10000], + } + ] + for operation, unit in ( + ("load", "load batches"), + ("offload", "owner offload nodes"), + ): + counters.update( + { + f"{operation}_operations_total": f"Enqueued {unit}", + f"{operation}_started_total": f"Dequeued {unit}, before event synchronization and preparation", + f"{operation}_completed_total": f"Finished {unit}, including failures; before ordered publication", + f"{operation}_errors_total": f"Failed {unit}, including preparation, submission and completion errors", + } + ) + for suffix, description in ( + ( + "queue_duration_ms", + "Enqueue to worker dequeue time; excludes scheduler pending-load residence", + ), + ( + "duration_ms", + "Enqueue to worker completion time including queueing, preparation and failures; excludes ordered-publication delay and downstream background persistence", + ), + ): + histograms.append( + { + "name": f"sglang_linker_{operation}_{suffix}", + "documentation": f"SGLang linker {operation}: {description} (ms)", + "buckets": [ + 0.01, + 0.1, + 0.5, + 1, + 5, + 10, + 50, + 100, + 500, + 1000, + 5000, + 10000, + ], + } + ) + return { + "counter": [ + {"name": f"sglang_linker_{name}", "documentation": f"SGLang linker: {doc}"} + for name, doc in counters.items() + ], + "histogram": histograms, + } + + +def get_default_sglang_metrics_config() -> dict: + """Return independent definitions without reading any YAML at runtime.""" + config = get_default_metrics_config() + # Standalone KV exporters use kv: aliases. SGLang's shared endpoint keeps + # the existing ucm: namespace used by its dashboards and explicit YAML. + for kind in ("counter", "gauge", "histogram"): + for definition in config[kind]: + if str(definition.get("prometheus_name", "")).startswith("kv:"): + definition.pop("prometheus_name") + config.pop("vllm_connector_prefix", None) + config["multiproc_prefix"] = "ucm:" + config["consumers"] = {"multiproc": True, "vllm_connector": False} + config["counter"] = [ + { + "name": "sglang_connector_load_requests_total", + "documentation": "SGLang connector load: Physical storage transfer calls attempted, per worker and pool or layer", + }, + { + "name": "sglang_connector_load_bytes_total", + "documentation": "SGLang connector load: Bytes transferred by successful storage calls, local to this worker", + }, + { + "name": "sglang_connector_load_submit_errors_total", + "documentation": "SGLang connector load: Storage submission failures", + }, + { + "name": "sglang_connector_load_wait_errors_total", + "documentation": "SGLang connector load: Storage completion wait failures", + }, + { + "name": "sglang_connector_dump_requests_total", + "documentation": "SGLang connector dump: Physical storage transfer calls attempted, per worker and pool or layer", + }, + { + "name": "sglang_connector_dump_bytes_total", + "documentation": "SGLang connector dump: Bytes transferred by successful storage calls, local to this worker", + }, + { + "name": "sglang_connector_dump_submit_errors_total", + "documentation": "SGLang connector dump: Storage submission failures", + }, + { + "name": "sglang_connector_dump_wait_errors_total", + "documentation": "SGLang connector dump: Storage completion wait failures", + }, + ] + config["counter"] + config["histogram"] = [ + { + "name": "sglang_connector_load_pages", + "documentation": "SGLang connector load: Successful storage batch size in pool pages; not deduplicated across TP/PP, pools or layers", + "buckets": [1, 5, 10, 50, 100, 128, 256, 512], + }, + { + "name": "sglang_connector_load_duration_ms", + "documentation": "SGLang connector load: Storage submission-to-completion duration including failed attempts (ms); excludes buffer preparation and restoration", + "buckets": [0.01, 0.1, 0.5, 1, 5, 10, 50, 100, 500, 1000, 5000, 10000], + }, + { + "name": "sglang_connector_load_bandwidth_gbps", + "documentation": "SGLang connector load: Successful call bytes divided by storage submission-to-completion time (GB/s, decimal bytes); not device-to-storage end-to-end bandwidth", + "buckets": [0.1, 0.5, 1, 5, 10, 20, 50, 100], + }, + { + "name": "sglang_connector_dump_pages", + "documentation": "SGLang connector dump: Successful storage batch size in pool pages; not deduplicated across TP/PP, pools or layers", + "buckets": [1, 5, 10, 50, 100, 128, 256, 512], + }, + { + "name": "sglang_connector_dump_duration_ms", + "documentation": "SGLang connector dump: Storage submission-to-completion duration including failed attempts (ms); excludes buffer preparation and restoration", + "buckets": [0.01, 0.1, 0.5, 1, 5, 10, 50, 100, 500, 1000, 5000, 10000], + }, + { + "name": "sglang_connector_dump_bandwidth_gbps", + "documentation": "SGLang connector dump: Successful call bytes divided by storage submission-to-completion time (GB/s, decimal bytes); not device-to-storage end-to-end bandwidth", + "buckets": [0.1, 0.5, 1, 5, 10, 20, 50, 100], + }, + ] + config["histogram"] + for kind, definitions in get_linker_metric_definitions().items(): + config[kind].extend(definitions) + return config diff --git a/ucm/integration/sglang/metrics.py b/ucm/integration/sglang/metrics.py new file mode 100644 index 000000000..2ca14ec17 --- /dev/null +++ b/ucm/integration/sglang/metrics.py @@ -0,0 +1,235 @@ +"""Per-worker storage-call observations; export uses the existing UCM logger.""" + +import logging +import os +import time +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Callable + +logger = logging.getLogger(__name__) + + +def create_metrics_logger(*, enabled, model_name, ranks, launch_config): + """Register once per worker, before Store construction, using SGLang's dir.""" + if not enabled: + return None + from ucm.metrics_config import ( + consumer_enabled, + get_metric_definitions, + load_metrics_config, + metrics_enabled, + ) + + if not metrics_enabled(launch_config): + return None + config_path = launch_config.get("metrics_config_path") + if config_path: + config = load_metrics_config(config_path) + else: + from ucm.integration.sglang.default_metrics_config import ( + get_default_sglang_metrics_config, + ) + + config = get_default_sglang_metrics_config() + if not get_metric_definitions(config): + raise ValueError(f"No UCM metrics defined in {config_path}") + if not consumer_enabled(config, "multiproc") or consumer_enabled( + config, "vllm_connector" + ): + raise ValueError( + "SGLang metrics config must enable multiproc and disable vllm_connector" + ) + directory = os.environ.get("PROMETHEUS_MULTIPROC_DIR") + if not directory or not Path(directory).is_dir(): + raise RuntimeError( + "SGLang must set PROMETHEUS_MULTIPROC_DIR before UCM metrics" + ) + from prometheus_client import values + + if not getattr(values.ValueClass, "_multiprocess", False): + raise RuntimeError( + "PROMETHEUS_MULTIPROC_DIR must be set before importing prometheus_client" + ) + from ucm.observability import PrometheusStatsLogger, _metric_mappings + + if _metric_mappings: + raise RuntimeError( + "UCM metrics already registered in this worker; only one metrics-enabled " + "connector is supported. Restart the worker before recreating it." + ) + worker_id = "_".join( + f"{name}{ranks.get(name + '_rank', 0)}" + for name in ("dp", "tp", "pp", "attn_cp") + ) + return PrometheusStatsLogger(model_name or "", worker_id, config=config) + + +def shutdown_metrics_logger(metrics_logger): + """Stop polling, then export the last deltas after all Store work has drained.""" + if metrics_logger is None: + return + metrics_logger.shutdown() + try: + dispatcher = metrics_logger.metrics_dispatcher + dispatcher.drain_to_consumers() + metrics_logger.update_stats(*dispatcher.get_stats_and_clear("multiproc")) + except Exception: + logger.warning("Failed to flush SGLang UCM metrics", exc_info=True) + + +def record_metrics(values): + """Business threads only update accumulators; export remains on the logger.""" + try: + from ucm.shared.metrics import ucmmetrics + + ucmmetrics.update_stats(values) + except Exception: + logger.warning("Failed to record SGLang UCM metrics", exc_info=True) + + +@dataclass +class AsyncIoObservation: + """One actual shard submission, retained until Wait confirms final status.""" + + is_set: bool + pages: int + byte_count: int + started: float + submitted: bool = False + failed: bool = False + check_failed: bool = False + finished: bool = False + + @classmethod + def begin(cls, enabled, is_set, pages, shard_size): + if not enabled or pages == 0: + return None + return cls(is_set, pages, pages * shard_size, time.perf_counter()) + + def fail_check(self): + self.failed = True + if not self.check_failed: + self.check_failed = True + operation = "dump" if self.is_set else "load" + record_metrics({f"sglang_linker_{operation}_check_errors_total": 1}) + + def finish(self, failed=False): + if self.finished: + return + self.finished = True + elapsed = max(time.perf_counter() - self.started, 0.0) + prefix = "sglang_connector_dump" if self.is_set else "sglang_connector_load" + values = { + f"{prefix}_requests_total": 1, + f"{prefix}_duration_ms": elapsed * 1000, + } + if failed or self.failed: + phase = "wait" if self.submitted else "submit" + values[f"{prefix}_{phase}_errors_total"] = 1 + else: + values[f"{prefix}_bytes_total"] = self.byte_count + values[f"{prefix}_pages"] = self.pages + if elapsed > 0: + values[f"{prefix}_bandwidth_gbps"] = self.byte_count / elapsed / 1e9 + record_metrics(values) + + +@dataclass +class LinkerOperation: + """One load batch or owner offload node, from enqueue to worker completion.""" + + kind: str + queued_at: float + finished: bool = False + + @classmethod + def begin(cls, enabled, kind): + if not enabled: + return None + observation = cls(kind, time.perf_counter()) + record_metrics({f"sglang_linker_{kind}_operations_total": 1}) + return observation + + def start(self): + record_metrics( + { + f"sglang_linker_{self.kind}_started_total": 1, + f"sglang_linker_{self.kind}_queue_duration_ms": max( + time.perf_counter() - self.queued_at, 0.0 + ) + * 1000, + } + ) + + def finish(self, success): + if self.finished: + return + self.finished = True + values = { + f"sglang_linker_{self.kind}_completed_total": 1, + f"sglang_linker_{self.kind}_duration_ms": max( + time.perf_counter() - self.queued_at, 0.0 + ) + * 1000, + } + if not success: + values[f"sglang_linker_{self.kind}_errors_total"] = 1 + record_metrics(values) + + +@dataclass +class IoObservation: + submitted: bool = False + failed: bool = False + + +@contextmanager +def measure_io(enabled: bool, is_set: bool, pages: int, byte_count: Callable[[], int]): + """Observe one physical v1 call or v2 pool call, never an inference batch. + + Duration covers submission through wait, including failed attempts. Page and + bandwidth histograms and byte counters contain successful transfers only. + Callers mark caught wait failures so existing return/exception behavior is + preserved. No native metrics import, clock read, or byte calculation occurs + when disabled. No metric collection/draining happens on the business thread. + """ + observation = IoObservation() + if not enabled or pages == 0: + yield observation + return + + started = time.perf_counter() + try: + yield observation + except BaseException: + observation.failed = True + raise + finally: + elapsed = max(time.perf_counter() - started, 0.0) + prefix = "sglang_connector_dump" if is_set else "sglang_connector_load" + values = { + f"{prefix}_requests_total": 1, + f"{prefix}_duration_ms": elapsed * 1000, + } + if observation.failed: + phase = "wait" if observation.submitted else "submit" + values[f"{prefix}_{phase}_errors_total"] = 1 + # Observability must not change the result of a storage operation. + try: + if not observation.failed: + size = int(byte_count()) + if size < 0: + raise ValueError("negative storage transfer size") + values[f"{prefix}_bytes_total"] = size + values[f"{prefix}_pages"] = pages + if elapsed > 0: + values[f"{prefix}_bandwidth_gbps"] = size / elapsed / 1e9 + from ucm.shared.metrics import ucmmetrics + + ucmmetrics.update_stats(values) + except Exception: + logger.warning( + "Failed to record SGLang connector I/O metrics", exc_info=True + ) diff --git a/ucm/integration/sglang/ucm_connector.py b/ucm/integration/sglang/ucm_connector.py index b8b036d1d..d4f11e46e 100644 --- a/ucm/integration/sglang/ucm_connector.py +++ b/ucm/integration/sglang/ucm_connector.py @@ -1,7 +1,7 @@ import hashlib import logging import re -from dataclasses import dataclass +from dataclasses import dataclass, field from pathlib import Path from types import SimpleNamespace from typing import TYPE_CHECKING, Any, Dict, List, Optional, Sequence @@ -10,6 +10,7 @@ import yaml from sglang.srt.distributed.parallel_state import get_world_group +from ucm.integration.sglang.metrics import create_metrics_logger, measure_io from ucm.store.factory_v1 import UcmConnectorFactoryV1 if TYPE_CHECKING: @@ -23,6 +24,18 @@ logger = logging.getLogger(__name__) +def _create_metrics_logger(storage_config, launch_config): + return create_metrics_logger( + enabled=getattr(storage_config, "enable_storage_metrics", False), + model_name=getattr(storage_config, "model_name", ""), + ranks={ + name + "_rank": getattr(storage_config, name + "_rank", 0) + for name in ("dp", "tp", "pp", "attn_cp") + }, + launch_config=launch_config, + ) + + def _load_extra_config_from_yaml(config_file: str) -> Dict[str, Any]: with Path(config_file).open("r", encoding="utf-8") as f: data = yaml.safe_load(f) or {} @@ -39,6 +52,7 @@ class UnifiedCacheStoreConfig: module_path: str name: str config: Dict[str, Any] + launch_config: Dict[str, Any] = field(default_factory=dict) def fixed_size_config( self, namespace: str, tensor_size: int, tensors_per_block: int = 1 @@ -140,7 +154,9 @@ def load_from_config( cfg["block_size"] = block_size cfg["stream_number"] = 8 - return UnifiedCacheStoreConfig(module_path=module_path, name=name, config=cfg) + return UnifiedCacheStoreConfig( + module_path=module_path, name=name, config=cfg, launch_config=dict(kvc) + ) class SglangUcmConnector: @@ -152,6 +168,8 @@ def __init__( storage_backends: List[str], ): self.store = store + self._metrics_logger = None + self._closed = False self.mem_pool_host = mem_pool_host self.storage_backends = storage_backends @@ -190,19 +208,35 @@ def from_hicache( ucm_store_config = UnifiedCacheStoreConfig.load_from_config( storage_config, mem_pool_host ) + # Register metrics before creating stores: their constructors can emit + # resource gauges. One exporter covers all lazily created v2 pools too. + metrics_logger = _create_metrics_logger( + storage_config, ucm_store_config.launch_config + ) store = None - if getattr(mem_pool_host, "kv_buffer", None) is not None: - store = UcmConnectorFactoryV1.create_connector( - ucm_store_config.name, - ucm_store_config.config, - ucm_store_config.module_path, + try: + if getattr(mem_pool_host, "kv_buffer", None) is not None: + store = UcmConnectorFactoryV1.create_connector( + ucm_store_config.name, + ucm_store_config.config, + ucm_store_config.module_path, + ) + connector = cls( + store, + mem_pool_host, + storage_config, + ucm_store_config.config["storage_backends"], ) - connector = cls( - store, - mem_pool_host, - storage_config, - ucm_store_config.config["storage_backends"], - ) + except BaseException: + try: + close = getattr(store, "close", None) + if callable(close): + close() + finally: + if metrics_logger is not None: + metrics_logger.shutdown() + raise + connector._metrics_logger = metrics_logger connector.ucm_store_config = ucm_store_config return connector @@ -371,17 +405,26 @@ def batch_io_v2( page_results = self._batch_io_flattened_pool(transfer, is_set) results[transfer.name] = page_results continue - keys, page_ptrs, _ = self._transfer_meta(transfer) + keys, page_ptrs, page_sizes = self._transfer_meta(transfer) page_results = [True] * len(keys) store, _ = self.pool_components[transfer.name][0] encoded = [self._component_key(key, transfer.name, 0) for key in keys] try: - task = ( - store.dump_data(encoded, [0] * len(keys), page_ptrs) - if is_set - else store.load_data(encoded, [0] * len(keys), page_ptrs) - ) - store.wait(task) + with measure_io( + self._metrics_logger is not None, + is_set, + len(keys), + lambda: sum( + int(size) for sizes in page_sizes for size in sizes if size > 0 + ), + ) as observation: + task = ( + store.dump_data(encoded, [0] * len(keys), page_ptrs) + if is_set + else store.load_data(encoded, [0] * len(keys), page_ptrs) + ) + observation.submitted = True + store.wait(task) except RuntimeError as exc: logger.error( "UnifiedCache %s failed for pool %s: %s", @@ -430,12 +473,19 @@ def _batch_io_flattened_pool( encoded = [self._component_key(key, transfer.name, 0) for key in keys] pointers = [[page.data_ptr()] for page in staging_pages] try: - task = ( - store.dump_data(encoded, [0] * len(keys), pointers) - if is_set - else store.load_data(encoded, [0] * len(keys), pointers) - ) - store.wait(task) + with measure_io( + self._metrics_logger is not None, + is_set, + len(keys), + lambda: sum(actual_sizes), + ) as observation: + task = ( + store.dump_data(encoded, [0] * len(keys), pointers) + if is_set + else store.load_data(encoded, [0] * len(keys), pointers) + ) + observation.submitted = True + store.wait(task) except RuntimeError as exc: logger.error( "UnifiedCache %s failed for flattened pool %s: %s", @@ -516,18 +566,33 @@ def batch_exists_v2(self, keys, pool_transfers=None, extra_info=None): return PoolTransferResult(final_pages, hit_counts, restorable) def close(self) -> None: - seen = set() - for store in [self.store] + [ - store - for components in self.pool_components.values() - for store, _ in components - ]: - if id(store) in seen: - continue - seen.add(id(store)) - close = getattr(store, "close", None) - if callable(close): - close() + if self._closed: + return + self._closed = True + first_error = None + try: + seen = set() + for store in [self.store] + [ + store + for components in self.pool_components.values() + for store, _ in components + ]: + if id(store) in seen: + continue + seen.add(id(store)) + close = getattr(store, "close", None) + if callable(close): + try: + close() + except Exception as error: + if first_error is None: + first_error = error + logger.exception("Failed to close UCM store") + finally: + if self._metrics_logger is not None: + self._metrics_logger.shutdown() + if first_error is not None: + raise first_error def _encode_key(self, key: str) -> bytes: return hashlib.md5(key.encode("utf-8")).digest() @@ -595,12 +660,22 @@ def batch_get_v1( ) store = self._require_primary_store() - task = store.load_data(key_list, shard_index_list, ptr_list) - try: - store.wait(task) - except RuntimeError as e: - logger.error(f"UnifiedCache load KVCache failed: {e}") - return [False] * len(keys) + with measure_io( + self._metrics_logger is not None, + False, + len(keys), + lambda: ( + len(keys) * self.page_size * self.mem_pool_host.get_size_per_token() + ), + ) as observation: + task = store.load_data(key_list, shard_index_list, ptr_list) + observation.submitted = True + try: + store.wait(task) + except RuntimeError as e: + observation.failed = True + logger.error(f"UnifiedCache load KVCache failed: {e}") + return [False] * len(keys) return [True] * len(keys) @@ -623,12 +698,22 @@ def batch_set_v1( ) store = self._require_primary_store() - task = store.dump_data(key_list, shard_index_list, ptr_list) - try: - store.wait(task) - except RuntimeError as e: - logger.error(f"UnifiedCache dump KVCache failed: {e}") - return [False] * len(keys) + with measure_io( + self._metrics_logger is not None, + True, + len(keys), + lambda: ( + len(keys) * self.page_size * self.mem_pool_host.get_size_per_token() + ), + ) as observation: + task = store.dump_data(key_list, shard_index_list, ptr_list) + observation.submitted = True + try: + store.wait(task) + except RuntimeError as e: + observation.failed = True + logger.error(f"UnifiedCache dump KVCache failed: {e}") + return [False] * len(keys) return [True] * len(keys) diff --git a/ucm/integration/sglang/ucm_direct_linker.py b/ucm/integration/sglang/ucm_direct_linker.py index 7588a8380..358a83c3c 100644 --- a/ucm/integration/sglang/ucm_direct_linker.py +++ b/ucm/integration/sglang/ucm_direct_linker.py @@ -32,6 +32,12 @@ from sglang.srt.runtime_context import get_model, get_parallel from sglang.srt.utils import freeze_gc, get_device_module +from ucm.integration.sglang.metrics import ( + LinkerOperation, + create_metrics_logger, + record_metrics, + shutdown_metrics_logger, +) from ucm.integration.sglang.ucm_direct_store_adapter import ( UcmDirectStoreAdapter, UcmLayerTransfer, @@ -91,6 +97,7 @@ class _PendingOffload: transfers: list[UcmLayerTransfer] ready_events: list[object] result: bool | None = None + observation: LinkerOperation | None = None class UcmDirectLinker(UnifiedCacheLinker): @@ -111,6 +118,8 @@ def __init__( adapter: UcmDirectStoreAdapter | None = None, ): del server_args + self._metrics_logger = None + self._metrics_enabled = False self.forward_stream = forward_stream self.page_size = params.page_size self._kv_allocator = params.token_to_kv_pool_allocator @@ -131,16 +140,19 @@ def __init__( # Scheduler-owned state; workers receive batches through _load_queue. self._pending_loads: dict[str, list[PoolTransfer]] = {} self._load_queue: Queue[ - tuple[int, dict[str, list[PoolTransfer]], list[object]] | None + tuple[ + int, dict[str, list[PoolTransfer]], list[object], LinkerOperation | None + ] + | None ] = Queue() self._completed_loads: Queue[list[str]] = Queue() self._requires_load_finish = hasattr( UnifiedCacheLinker, "finish_layer_wise_loading" ) self._load_results: dict[int, Future[bool]] = {} - self._offload_queue: Queue[tuple[list[PoolTransfer], list[object]] | None] = ( - Queue() - ) + self._offload_queue: Queue[ + tuple[list[PoolTransfer], list[object], LinkerOperation | None] | None + ] = Queue() self._offload_results: Queue[bool] = Queue() self._lookup_progress: OrderedDict[str, _LookupProgress] = OrderedDict() @@ -158,9 +170,41 @@ def __init__( self._log_pool_topology() try: self.adapter = adapter or self._create_adapter(params, extra_config) + self._init_metrics(params, extra_config) self.adapter.register_device_pools(self._pool_metadata()) self._select_store_ranks(tp_group, params.attn_cp_size) + self.layer_done_counter = LayerWiseLoadCounter(self.num_layers) + if PoolName.MAMBA in self.pools: + params.req_to_token_pool.register_layer_transfer_counter( + self.layer_done_counter + ) + + self._load_thread = threading.Thread( + target=self._load_worker, + daemon=True, + name=f"ucm-load-tp{self._tp_rank}", + ) + self._offload_thread = threading.Thread( + target=self._offload_worker, + daemon=True, + name=f"ucm-offload-tp{self._tp_rank}", + ) + self._load_thread.start() + self._offload_thread.start() except Exception: + for thread_name, queue in ( + ("_load_thread", self._load_queue), + ("_offload_thread", self._offload_queue), + ): + thread = getattr(self, thread_name, None) + if thread is not None and thread.is_alive(): + queue.put(None) + thread.join() + try: + if hasattr(self, "adapter"): + self.adapter.close() + finally: + shutdown_metrics_logger(self._metrics_logger) logger.exception( "UCM linker initialization failed: layers=%d tp=%d/%d replicated=%s.", self.num_layers, @@ -170,22 +214,44 @@ def __init__( ) raise - self.layer_done_counter = LayerWiseLoadCounter(self.num_layers) - if PoolName.MAMBA in self.pools: - params.req_to_token_pool.register_layer_transfer_counter( - self.layer_done_counter + def _init_metrics(self, params, extra_config): + launch_config = dict(getattr(self.adapter, "launch_config", {})) + for key in ("enable_metrics", "metrics_config_path"): + if key in (extra_config or {}): + launch_config[key] = extra_config[key] + self._metrics_logger = create_metrics_logger( + enabled=getattr(params, "enable_metrics", False), + model_name=getattr(get_model(), "model_path", ""), + ranks={ + "dp_rank": ( + self._attn_dp_rank + if getattr(get_parallel(), "enable_dp_attention", False) + else self._dp_rank + ), + "tp_rank": self._tp_rank, + "pp_rank": params.pp_rank, + "attn_cp_rank": params.attn_cp_rank, + }, + launch_config=launch_config, + ) + self._metrics_enabled = self._metrics_logger is not None + self.adapter.metrics_enabled = self._metrics_enabled + if self._metrics_enabled: + # Initialize counter series so queue-depth differences and zero-error + # rates exist before the first completion, without fabricating samples. + record_metrics( + { + item["name"]: 0 + for item in self._metrics_logger.config.get("counter", []) + if item["name"].startswith("sglang_") + } ) - self._load_thread = threading.Thread( - target=self._load_worker, daemon=True, name=f"ucm-load-tp{self._tp_rank}" - ) - self._offload_thread = threading.Thread( - target=self._offload_worker, - daemon=True, - name=f"ucm-offload-tp{self._tp_rank}", - ) - self._load_thread.start() - self._offload_thread.start() + def _record_metrics(self, **values): + if self._metrics_enabled: + record_metrics( + {f"sglang_linker_{key}": value for key, value in values.items()} + ) def _select_store_ranks(self, tp_group: Any, attn_cp_size: int) -> None: """Find shared-Cache peers for lookup deduplication and dump partitioning.""" @@ -599,6 +665,7 @@ def lookup(self, rid: str, transfers: list[PoolTransfer]) -> list[int]: # SGLang intersects each rank's restorable-boundary set. Supplying # every boundary is neutral while the local host owner probes its # shared Cache/Posix view. + self._record_metrics(lookup_skipped_total=1) return list(range(1, len(page_keys) + 1)) lookup_transfers = self._page_transfers(expanded, for_io=False) if not lookup_transfers: @@ -627,11 +694,30 @@ def lookup(self, rid: str, transfers: list[PoolTransfer]) -> list[int]: ) ): resume_from_pages = previous.hit_pages - result = self.adapter.lookup( - lookup_transfers, - page_keys, - resume_from_pages=resume_from_pages, - ) + started = time.perf_counter() if self._metrics_enabled else None + lookup_succeeded = False + try: + result = self.adapter.lookup( + lookup_transfers, + page_keys, + resume_from_pages=resume_from_pages, + ) + lookup_succeeded = True + finally: + if started is not None: + # Capture adapter latency before constructing or recording metrics. + values = { + "lookup_duration_ms": (time.perf_counter() - started) * 1000, + "lookup_requests_total": 1, + "lookup_resumed_total": int(resume_from_pages > 0), + } + if lookup_succeeded: + # Local restorable boundary, not final cross-rank token reuse. + values["lookup_query_pages_total"] = len(page_keys) + values["lookup_hit_pages_total"] = result[-1] if result else 0 + else: + values["lookup_errors_total"] = 1 + self._record_metrics(**values) self._lookup_progress[rid] = _LookupProgress( tuple(page_keys), transfer_keys, result[-1] if result else 0 ) @@ -648,10 +734,14 @@ def load(self, rid: str, transfers: list[PoolTransfer]) -> bool: if rid in self._pending_loads: raise RuntimeError(f"UCM load for rid={rid} is already queued.") self._pending_loads[rid] = expanded + self._record_metrics(load_requests_total=1) return True def cancel_queued_load(self, rid: str) -> bool: - return self._pending_loads.pop(rid, None) is not None + cancelled = self._pending_loads.pop(rid, None) is not None + if cancelled: + self._record_metrics(load_cancelled_requests_total=1) + return cancelled def num_completed_loads(self) -> int: return self._completed_loads.qsize() @@ -703,7 +793,8 @@ def start_layer_wise_loading(self) -> int: if self._requires_load_finish: self._load_results[counter_index] = Future() ready_events = self._record_transfer_events() - self._load_queue.put((counter_index, pending, ready_events)) + observation = LinkerOperation.begin(self._metrics_enabled, "load") + self._load_queue.put((counter_index, pending, ready_events, observation)) return counter_index def finish_layer_wise_loading(self, counter_index: int) -> bool: @@ -722,7 +813,9 @@ def _load_worker(self) -> None: try: if task is None: return - counter_index, pending, ready_events = task + counter_index, pending, ready_events, observation = task + if observation is not None: + observation.start() landed = True active_layer = -1 try: @@ -756,6 +849,7 @@ def _load_worker(self) -> None: self.layer_done_counter.complete( counter_index, next_complete ) + self._record_metrics(load_ready_layers_total=1) next_complete += 1 while next_complete < self.num_layers: @@ -764,6 +858,7 @@ def _load_worker(self) -> None: self.layer_done_counter.complete( counter_index, next_complete ) + self._record_metrics(load_ready_layers_total=1) next_complete += 1 finally: # A failed submit or wait must not leave prefetched @@ -787,6 +882,8 @@ def _load_worker(self) -> None: len(pending), ) finally: + if observation is not None: + observation.finish(landed) if self._requires_load_finish: self._load_results[counter_index].set_result(landed) else: @@ -868,11 +965,13 @@ def offload( self._freeze_gc_once() selected = self._partition_offload(expanded) if not selected: + self._record_metrics(offload_skipped_total=1) # Empty ranks still occupy their position in the node-result FIFO. - self._offload_queue.put(([], [])) + self._offload_queue.put(([], [], None)) return True ready_events = self._record_transfer_events() - self._offload_queue.put((selected, ready_events)) + observation = LinkerOperation.begin(self._metrics_enabled, "offload") + self._offload_queue.put((selected, ready_events, observation)) return True def _offload_worker(self) -> None: @@ -883,13 +982,15 @@ def _offload_worker(self) -> None: next_poll_at = 0.0 sequence = 0 - def submit(task: tuple[list[PoolTransfer], list[object]]) -> None: + def submit(task) -> None: nonlocal sequence sequence += 1 - expanded, ready_events = task + expanded, ready_events, observation = task if not expanded: in_flight.append(_PendingOffload(sequence, [], [], [], result=True)) return + if observation is not None: + observation.start() transfers: list[UcmLayerTransfer] = [] try: prerequisite_handle = self._wait_transfer_events(ready_events) @@ -898,6 +999,8 @@ def submit(task: tuple[list[PoolTransfer], list[object]]) -> None: transfers, prerequisite_handle=prerequisite_handle ) except BaseException: + if observation is not None: + observation.finish(False) logger.exception( "UCM offload submit failed tp_rank=%d node=%d", self._tp_rank, @@ -921,6 +1024,7 @@ def submit(task: tuple[list[PoolTransfer], list[object]]) -> None: tasks=tasks, transfers=transfers, ready_events=ready_events, + observation=observation, ) ) @@ -975,6 +1079,8 @@ def submit(task: tuple[list[PoolTransfer], list[object]]) -> None: pending.sequence, ) pending.result = False + if pending.observation is not None: + pending.observation.finish(pending.result) pending.tasks = [] pending.transfers = [] pending.ready_events = [] @@ -1003,6 +1109,7 @@ def _drain(queue: Queue) -> None: def reset(self) -> None: self._lookup_progress.clear() + self._record_metrics(load_cancelled_requests_total=len(self._pending_loads)) self._pending_loads.clear() self._load_queue.join() self._load_results.clear() @@ -1020,4 +1127,7 @@ def close(self) -> None: self._offload_queue.put(None) self._load_thread.join() self._offload_thread.join() - self.adapter.close() + try: + self.adapter.close() + finally: + shutdown_metrics_logger(self._metrics_logger) diff --git a/ucm/integration/sglang/ucm_direct_store_adapter.py b/ucm/integration/sglang/ucm_direct_store_adapter.py index cd8be724c..447ffd3ea 100644 --- a/ucm/integration/sglang/ucm_direct_store_adapter.py +++ b/ucm/integration/sglang/ucm_direct_store_adapter.py @@ -15,6 +15,7 @@ import numpy as np import yaml +from ucm.integration.sglang.metrics import AsyncIoObservation, record_metrics from ucm.store.factory_v1 import UcmConnectorFactoryV1 from ucm.store.ucmstore_v1 import Task, UcmKVStoreBaseV1 @@ -56,6 +57,7 @@ class _StoreTask: task: Task # Keep address arrays alive until the native task is reaped. addresses: np.ndarray + observation: AsyncIoObservation | None = None @dataclass(frozen=True) @@ -135,6 +137,8 @@ def __init__( identity: dict[str, Any], ) -> None: self._connector_name = connector_name + self.metrics_enabled = False + self.launch_config: dict[str, Any] = {} self._module_path = module_path self._identity = dict(identity) self._block_identity_prefix = self._identity_suffix() @@ -190,12 +194,20 @@ def from_config( raise ValueError("UCM connector entry is missing ucm_connector_name.") if not isinstance(store_config, dict): raise ValueError("UCM connector entry is missing ucm_connector_config.") - return cls( + adapter = cls( connector_name=name, module_path=connector.get("ucm_connector_module_path"), store_config=store_config, identity=identity, ) + # Metrics options live beside ucm_connectors, never in a Store config. + # Explicit linker options override the YAML, including false values. + adapter.launch_config = { + key: config.get(key, document.get(key)) + for key in ("enable_metrics", "metrics_config_path") + if key in config or key in document + } + return adapter def _estimate_gc_capacities( self, by_group: dict[str, list[UcmPoolMetadata]] @@ -233,7 +245,7 @@ def _estimate_gc_capacities( capacities[group] += 1 for group in weights: logger.info( - "SGLang UCM GC capacity group=%s estimated_bytes=%d " "capacity_gb=%d", + "SGLang UCM GC capacity group=%s estimated_bytes=%d capacity_gb=%d", group, weights[group], capacities[group], @@ -611,6 +623,8 @@ def lookup( f"invalid prefix hit count={found} pages={len(query_keys)}" ) if start and found == 0: + if self.metrics_enabled: + record_metrics({"sglang_linker_lookup_fallback_total": 1}) # The previous boundary vanished; rebuild from page 0. found = ( store.lookup_on_prefix(self._block_ids(transfer, keys)) + 1 @@ -775,26 +789,36 @@ def _submit_dump( addresses = transfer.addresses.reshape(-1, len(transfer.tensor_size_list)) if not len(addresses): return - if item.prepared_io is not None: - task = item.prepared_io.dump_data(addresses, prerequisite_handle) - else: - # Keep custom Stores on their public interface. Native Pipeline - # consumes these descriptors before returning the task handle. - page_count = len(item.block_ids) - rows = range(len(addresses)) - block_ids = [item.block_ids[row % page_count] for row in rows] - indexes = [row // page_count for row in rows] - task = ( - item.store.dump_data( - block_ids, - indexes, - addresses, - prerequisite_handle=prerequisite_handle, + observation = AsyncIoObservation.begin( + self.metrics_enabled, True, len(addresses), transfer.shard_size + ) + try: + if item.prepared_io is not None: + task = item.prepared_io.dump_data(addresses, prerequisite_handle) + else: + # Keep custom Stores on their public interface. Native Pipeline + # consumes these descriptors before returning the task handle. + page_count = len(item.block_ids) + rows = range(len(addresses)) + block_ids = [item.block_ids[row % page_count] for row in rows] + indexes = [row // page_count for row in rows] + task = ( + item.store.dump_data( + block_ids, + indexes, + addresses, + prerequisite_handle=prerequisite_handle, + ) + if prerequisite_handle + else item.store.dump_data(block_ids, indexes, addresses) ) - if prerequisite_handle - else item.store.dump_data(block_ids, indexes, addresses) - ) - tasks.append(_StoreTask(item.store, task, addresses)) + except BaseException: + if observation is not None: + observation.finish(failed=True) + raise + if observation is not None: + observation.submitted = True + tasks.append(_StoreTask(item.store, task, addresses, observation)) def _submit_prepared( self, @@ -843,9 +867,22 @@ def _submit_prepared( addresses, ) ) - task = operation(*args, **transfer_options) - tasks.append(_StoreTask(item.store, task, addresses)) - except Exception: + observation = AsyncIoObservation.begin( + self.metrics_enabled, + is_offload, + len(transfer.page_keys), + transfer.shard_size, + ) + try: + task = operation(*args, **transfer_options) + except BaseException: + if observation is not None: + observation.finish(failed=True) + raise + if observation is not None: + observation.submitted = True + tasks.append(_StoreTask(item.store, task, addresses, observation)) + except BaseException: # A later layer, batch or group can fail after earlier tasks # started. Reap those handles before their device buffers can # be reused by the caller. @@ -892,18 +929,28 @@ def submit_offload( def wait(self, tasks: list[_StoreTask]) -> None: # Wait consumes task handles; do not Check them again after this call. - first_error: Exception | None = None + first_error: BaseException | None = None for item in tasks: + failed = False try: item.store.wait(item.task) - except Exception as error: + except BaseException as error: + failed = True if first_error is None: first_error = error + finally: + if item.observation is not None: + item.observation.finish(failed=failed) if first_error is not None: raise first_error def check(self, task: _StoreTask) -> bool: - return bool(task.store.check(task.task)) + try: + return bool(task.store.check(task.task)) + except BaseException: + if task.observation is not None: + task.observation.fail_check() + raise def close(self) -> None: with self._store_lock: diff --git a/ucm/observability.py b/ucm/observability.py index e5fe6a402..a14c2ce3c 100644 --- a/ucm/observability.py +++ b/ucm/observability.py @@ -26,6 +26,7 @@ import os import threading import time +from copy import deepcopy from typing import Any, Union from prometheus_client import Counter, Gauge, Histogram @@ -52,15 +53,17 @@ class PrometheusStatsLogger: - def __init__(self, model_name, worker_id, config_path): + def __init__(self, model_name, worker_id, config_path=None, *, config=None): """ - Load metrics config from YAML file (config_path), + Use an in-memory config, or load a YAML file (config_path), register metrics using prometheus_client, and start a thread to get updated metrics. """ if _metric_mappings: logger.warning("Metrics are already registered, skipping re-registration.") return - self.config = load_metrics_config(config_path) + self.config = ( + deepcopy(config) if config is not None else load_metrics_config(config_path) + ) self.metric_definitions = get_metric_definitions(self.config) if not self.metric_definitions: return