Skip to main content

NVIDIA DCGM Exporter (Prometheus)

Description​

This connector scrapes NVIDIA dcgm-exporter Prometheus metrics over HTTP(S).

gpu hardware nvidia

Enterprise Connector

This connector requires the Enterprise edition of MetricsHub.

Target​

Typical platform: Nvidia

Operating systems: Linux, Out-Of-Band

Prerequisites​

Leverages: NVIDIA DCGM Exporter

Technology and protocols: N/A

Examples​

CLI​

metricshub HOSTNAME -t linux -c +NvidiaDCGMExporter

metricshub.yaml​

resourceGroups:
<RESOURCE_GROUP>:
resources:
<HOSTNAME-ID>:
attributes:
host.name: <HOSTNAME> # Change with actual host name
host.type: linux
connectors: [ +NvidiaDCGMExporter ] # Optional, to load only this connector
protocols:

Connector Activation Criteria​

The NVIDIA DCGM Exporter (Prometheus) connector will be automatically activated, and its status will be reported as OK if all the below criteria are met:

  • The HTTP Request below to the managed host succeeds:
    • get /metrics
    • The response body contains: DCGM_FI_DEV_GPU_UTIL (regex)

Metrics​

TypeCollected MetricsSpecific Attributes
gpuhw.energy{hw.type="gpu"}
hw.errors{hw.type="gpu", hw.error.type="ecc-memory-corrected", hw.error.severity="non-critical"}
hw.errors{hw.type="gpu", hw.error.type="ecc-memory-uncorrected", hw.error.severity="critical"}
hw.errors{hw.type="gpu", hw.error.type="nvlink-crc"}
hw.errors{hw.type="gpu", hw.error.type="nvlink-crc-flit"}
hw.errors{hw.type="gpu", hw.error.type="nvlink-recovery"}
hw.errors{hw.type="gpu", hw.error.type="nvlink-replay"}
hw.errors{hw.type="gpu", hw.error.type="pcie-replay"}
hw.errors{hw.type="gpu", hw.error.type="xid"}
hw.gpu.bandwidth.limit{interface="nvlink", limit_type="max"}
hw.gpu.bar1.limit
hw.gpu.bar1.usage{hw.gpu.bar1.state="free"}
hw.gpu.bar1.usage{hw.gpu.bar1.state="used"}
hw.gpu.clock.limit{hw.type="memory"}
hw.gpu.clock.limit{hw.type="sm"}
hw.gpu.clock.limit{hw.type="video"}
hw.gpu.clock.reduction.time{hw.gpu.clock.reference="application"}
hw.gpu.clock.reduction.time{hw.gpu.clock.reference="base"}
hw.gpu.clock{hw.type="memory"}
hw.gpu.clock{hw.type="sm"}
hw.gpu.clock{hw.type="video"}
hw.gpu.fabric.condition.active{hw.gpu.fabric.condition.type="access-timeout-recovery"}
hw.gpu.fabric.condition.active{hw.gpu.fabric.condition.type="bandwidth-degraded"}
hw.gpu.fabric.condition.active{hw.gpu.fabric.condition.type="route-recovery"}
hw.gpu.fabric.condition.active{hw.gpu.fabric.condition.type="route-unhealthy"}
hw.gpu.fabric.configuration.status{state="gpu-state-invalid|incompatible-gpu-firmware|incorrect-chassis-serial-number|incorrect-system-guid|insufficient-nvlinks|invalid-location|no-partition|ok"}
hw.gpu.fabric.partition.assigned
hw.gpu.io{direction="receive", interface="nvlink"}
hw.gpu.io{direction="receive", interface="pcie"}
hw.gpu.io{direction="transmit", interface="nvlink"}
hw.gpu.io{direction="transmit", interface="pcie"}
hw.gpu.io{interface="nvlink"}
hw.gpu.memory.limit
hw.gpu.memory.usage{hw.gpu.memory.state="free"}
hw.gpu.memory.usage{hw.gpu.memory.state="reserved"}
hw.gpu.memory.usage{hw.gpu.memory.state="used"}
hw.gpu.memory.utilization
hw.gpu.nvlink.io{direction="receive", nvlink="0"}
hw.gpu.nvlink.io{direction="receive", nvlink="1"}
hw.gpu.nvlink.io{direction="receive", nvlink="10"}
hw.gpu.nvlink.io{direction="receive", nvlink="11"}
hw.gpu.nvlink.io{direction="receive", nvlink="12"}
hw.gpu.nvlink.io{direction="receive", nvlink="13"}
hw.gpu.nvlink.io{direction="receive", nvlink="14"}
hw.gpu.nvlink.io{direction="receive", nvlink="15"}
hw.gpu.nvlink.io{direction="receive", nvlink="16"}
hw.gpu.nvlink.io{direction="receive", nvlink="17"}
hw.gpu.nvlink.io{direction="receive", nvlink="2"}
hw.gpu.nvlink.io{direction="receive", nvlink="3"}
hw.gpu.nvlink.io{direction="receive", nvlink="4"}
hw.gpu.nvlink.io{direction="receive", nvlink="5"}
hw.gpu.nvlink.io{direction="receive", nvlink="6"}
hw.gpu.nvlink.io{direction="receive", nvlink="7"}
hw.gpu.nvlink.io{direction="receive", nvlink="8"}
hw.gpu.nvlink.io{direction="receive", nvlink="9"}
hw.gpu.nvlink.io{direction="transmit", nvlink="0"}
hw.gpu.nvlink.io{direction="transmit", nvlink="1"}
hw.gpu.nvlink.io{direction="transmit", nvlink="10"}
hw.gpu.nvlink.io{direction="transmit", nvlink="11"}
hw.gpu.nvlink.io{direction="transmit", nvlink="12"}
hw.gpu.nvlink.io{direction="transmit", nvlink="13"}
hw.gpu.nvlink.io{direction="transmit", nvlink="14"}
hw.gpu.nvlink.io{direction="transmit", nvlink="15"}
hw.gpu.nvlink.io{direction="transmit", nvlink="16"}
hw.gpu.nvlink.io{direction="transmit", nvlink="17"}
hw.gpu.nvlink.io{direction="transmit", nvlink="2"}
hw.gpu.nvlink.io{direction="transmit", nvlink="3"}
hw.gpu.nvlink.io{direction="transmit", nvlink="4"}
hw.gpu.nvlink.io{direction="transmit", nvlink="5"}
hw.gpu.nvlink.io{direction="transmit", nvlink="6"}
hw.gpu.nvlink.io{direction="transmit", nvlink="7"}
hw.gpu.nvlink.io{direction="transmit", nvlink="8"}
hw.gpu.nvlink.io{direction="transmit", nvlink="9"}
hw.gpu.occupancy
hw.gpu.pcie.link.generation
hw.gpu.pcie.link.generation.limit{limit_type="max"}
hw.gpu.pcie.link.width
hw.gpu.pcie.link.width.limit{limit_type="max"}
hw.gpu.performance_state
hw.gpu.throttle.time{hw.gpu.throttle.reason="board-limit"}
hw.gpu.throttle.time{hw.gpu.throttle.reason="hardware-power-brake"}
hw.gpu.throttle.time{hw.gpu.throttle.reason="hardware-thermal"}
hw.gpu.throttle.time{hw.gpu.throttle.reason="low-utilization"}
hw.gpu.throttle.time{hw.gpu.throttle.reason="reliability"}
hw.gpu.throttle.time{hw.gpu.throttle.reason="software-power-cap"}
hw.gpu.throttle.time{hw.gpu.throttle.reason="software-thermal"}
hw.gpu.throttle.time{hw.gpu.throttle.reason="sync-boost"}
hw.gpu.utilization{hw.gpu.task="decoder"}
hw.gpu.utilization{hw.gpu.task="dram"}
hw.gpu.utilization{hw.gpu.task="encoder"}
hw.gpu.utilization{hw.gpu.task="fp16"}
hw.gpu.utilization{hw.gpu.task="fp32"}
hw.gpu.utilization{hw.gpu.task="fp64"}
hw.gpu.utilization{hw.gpu.task="general"}
hw.gpu.utilization{hw.gpu.task="graphics-engine"}
hw.gpu.utilization{hw.gpu.task="integer"}
hw.gpu.utilization{hw.gpu.task="memory-copy"}
hw.gpu.utilization{hw.gpu.task="sm"}
hw.gpu.utilization{hw.gpu.task="tensor"}
hw.power.limit{hw.type="gpu"}
hw.power{hw.type="gpu"}
hw.status{hw.type="gpu", hw.gpu.subsystem="fabric", state="degraded|failed|ok"}
hw.status{hw.type="gpu", hw.gpu.subsystem="fabric-manager", state="degraded|failed|ok"}
hw.status{hw.type="gpu", state="present"}
hw.temperature{sensor_location="gpu"}
hw.temperature{sensor_location="gpu-memory"}
host.name
hw.gpu.fabric.clique.id
hw.gpu.fabric.cluster.uuid
hw.gpu.index
hw.gpu.model
hw.gpu.nvidia.mig_instance_id
hw.gpu.pci.bus_id
hw.gpu.uuid
hw.id
hw.name
id
name
type
networkhw.errors{hw.type="network", hw.error.type="crc"}
hw.errors{hw.type="network", hw.error.type="ecc"}
hw.errors{hw.type="network", hw.error.type="flit"}
hw.errors{hw.type="network", hw.error.type="recovery"}
hw.errors{hw.type="network", hw.error.type="replay"}
hw.network.bandwidth.limit
hw.network.io{direction="receive"}
hw.network.io{direction="transmit"}
hw.network.up
hw.status{hw.type="network", state="degraded|failed|ok"}
hw.status{hw.type="network", state="present"}
device_type
host.name
hw.id
hw.name
hw.parent.id
hw.parent.type
id
name
other_devicehw.power{hw.type="other_device"}
hw.status{hw.type="other_device", state="degraded|failed|ok"}
hw.status{hw.type="other_device", state="present"}
device_type
host.name
hw.id
hw.name
id
name
temperaturehw.status{hw.type="temperature", state="present"}
hw.temperature
hw.temperature.limit{limit_type="high.critical"}
hw.temperature.limit{limit_type="high.degraded"}
host.name
hw.id
hw.name
hw.parent.id
hw.parent.type
id
name