Skip to content

HDDS-15396. Grafana dashboard for disk balancer #10609

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
navinko wants to merge 4 commits into
base: master
Choose a base branch
from
Draft

HDDS-15396. Grafana dashboard for disk balancer #10609

Changes from 1 commit
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
38bde60
HDDS-15396. Added new grafana dashboard for disk balancer
navinko Jun 25, 2026
a912e88
HDDS-15396. Addressed review comments
navinko Jun 28, 2026
bbedcf7
HDDS-15396. Addressed review comments
navinko Jul 18, 2026
f18488b
Merge branch 'apache:master' into HDDS-15396
navinko Jul 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
358 changes: 358 additions & 0 deletions hadoop-ozone/dist/src/main/compose/common/grafana/dashboards/Ozone - Disk Balancer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,358 @@
{
"annotations": {
"list": [
{
"builtIn": true,
"enable": true,
"hide": true,
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"liveNow": false,
"panels": [
{
"id": 2,
"title": "Disk Space Convergence Profile (Historical Before vs After)",
"type": "timeseries",
"description": "Shows how full each disk is over time. When the balancer is running, you should see these lines slowly move closer together until they meet in the middle.",
"gridPos": { "x": 0, "y": 0, "w": 12, "h": 7 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{
"expr": "{__name__=~\"volume_info_metrics_data_disk[0-9]+_ozone_used\", instance=~\"$datanode.*\"}",
Comment thread
Gargi-jais11 marked this conversation as resolved.
Outdated
"legendFormat": "{{storagedirectory}}",

@Gargi-jais11 Gargi-jais11 Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With multiple datanodes selected, duplicate paths from different hosts are indistinguishable.

Suggested change
"legendFormat": "{{storagedirectory}}",
"legendFormat": "{{instance}} {{storagedirectory}}",

"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "decbytes",
"color": { "mode": "palette-classic" },
"custom": {
"drawStyle": "line",
"lineInterpolation": "smooth",
"lineWidth": 2
}
}
}
},
{
"id": 3,
"title": "Current Disk Space Allocation Skew (Max - Min Volume Gap)",
"type": "bargauge",
"description": "The exact size of the data gap between your most full disk and your emptiest disk right now. The balancer's goal is to shrink this number down to zero.",
"gridPos": { "x": 12, "y": 0, "w": 12, "h": 7 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{
"expr": "max({__name__=~\"volume_info_metrics_data_disk[0-9]+_ozone_used\", instance=~\"$datanode.*\"}) by (instance) - min({__name__=~\"volume_info_metrics_data_disk[0-9]+_ozone_used\", instance=~\"$datanode.*\"}) by (instance)",
"instant": true,
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "decbytes",
"max": 5368709120
Comment thread
Gargi-jais11 marked this conversation as resolved.
Outdated
}
},
"options": {
"displayMode": "lcd",
"orientation": "horizontal",
"reduceOptions": { "calcs": ["lastNotNull"] }
}
},
{
"id": 5,
"title": "Active Balancing Throughput Speed",
"type": "stat",
"description": "The live speedometer. This shows exactly how fast data is physically copying from one disk to another across the node.",
"gridPos": { "x": 0, "y": 7, "w": 6, "h": 7 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{
"expr": "rate(disk_balancer_service_metrics_success_bytes{instance=~\"$datanode.*\"}[1m])",
"refId": "A"
}
],
Comment on lines +352 to +356

@Gargi-jais11 Gargi-jais11 Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

better to break it in this way.

Suggested change
"expr": "{__name__=~\"disk_balancer_service_metrics_move_success_time_avg_time|disk_balancer_service_metrics_move_failure_time_avg_time\", instance=~\"$datanode\"}",
"legendFormat": "{{__name__}}",
"refId": "A"
}
],
"expr": "disk_balancer_service_metrics_move_success_time_avg_time{instance=~\"$datanode\"}",
"legendFormat": "{{instance}} success avg latency",
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"expr": "disk_balancer_service_metrics_move_failure_time_avg_time{instance=~\"$datanode\"}",
"legendFormat": "{{instance}} failure avg latency",
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"expr": "disk_balancer_service_metrics_move_success_time_num_ops{instance=~\"$datanode\"}",
"legendFormat": "{{instance}} moves/interval",
"refId": "C"
}
],

"fieldConfig": {
"defaults": {
"unit": "Bps"
}
},
"options": {
"graphMode": "area",
"textMode": "value",
"reduceOptions": { "calcs": ["lastNotNull"] }
}
},
{
"id": 7,
"title": "Unscheduled Target Backlog",
"type": "stat",
"description": "The remaining amount of data the balancer still needs to queue up to reach perfect balance.",
"gridPos": { "x": 6, "y": 7, "w": 8, "h": 7 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{
"expr": "max({__name__=~\"volume_info_metrics_data_disk[0-9]+_ozone_used\", instance=~\"$datanode.*\"}) by (instance) - avg({__name__=~\"volume_info_metrics_data_disk[0-9]+_ozone_used\", instance=~\"$datanode.*\"}) by (instance)",
Comment thread
Gargi-jais11 marked this conversation as resolved.
Outdated
"instant": true,
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "decbytes"
}
},
"options": {
"graphMode": "none",
"textMode": "value",
"reduceOptions": { "calcs": ["lastNotNull"] }
}
},
{
"id": 8,
"title": "Est. Time for Unscheduled Backlog",
"type": "stat",
"description": "A rough estimate of how many minutes are left before the balancer finishes balancing.",
"gridPos": { "x": 14, "y": 7, "w": 10, "h": 7 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{
"expr": "((sum(max({__name__=~\"volume_info_metrics_data_disk[0-9]+_ozone_used\", instance=~\"$datanode.*\"}) by (instance) - avg({__name__=~\"volume_info_metrics_data_disk[0-9]+_ozone_used\", instance=~\"$datanode.*\"}) by (instance)) / (sum(rate(disk_balancer_service_metrics_success_bytes{instance=~\"$datanode.*\"}[1m])) > 0)) / 60) or vector(0)",
Comment thread
Gargi-jais11 marked this conversation as resolved.
Outdated
"instant": true,
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "m"

@Gargi-jais11 Gargi-jais11 Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If diskbalancer is in runnign state but not actively balancing than showing "0mins is ambigous"

Suggested change
"unit": "m"
"unit": "m",
"noValue": "N/A"

}
},
"options": {
"graphMode": "none",
"textMode": "value",
"reduceOptions": { "calcs": ["lastNotNull"] }
}
},
{
"id": 6,
"title": "Container Migration Latency Profile (Processing Duration)",
"type": "timeseries",
"description": "How much time it takes the system to successfully copy, verify, and finalize a single container.",
"gridPos": { "x": 0, "y": 14, "w": 24, "h": 7 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{
"expr": "{__name__=~\"disk_balancer_service_metrics_copy_success_latency|disk_balancer_service_metrics_copy_failure_latency|disk_balancer_service_metrics_move_success_time_avg_time\", instance=~\"$datanode.*\"}",
Comment thread
Gargi-jais11 marked this conversation as resolved.
Outdated
"legendFormat": "{{__name__}}",
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "ms",
"color": { "mode": "palette-classic" }
}
}
},
{
"id": 11,
"title": "Overall Move Success Rate",
Comment thread
Gargi-jais11 marked this conversation as resolved.
Outdated
"type": "gauge",
"description": "The percentage of container moves that finished perfectly.",
"gridPos": { "x": 0, "y": 21, "w": 5, "h": 6 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{
"expr": "(sum(disk_balancer_service_metrics_success_count) / (sum(disk_balancer_service_metrics_success_count) + sum(disk_balancer_service_metrics_failure_count) + 0.00001)) * 100",
"instant": true,
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"unit": "percent",
"min": 0,
"max": 100,
"thresholds": {
"mode": "absolute",
"steps": [
{ "value": 0, "color": "green" },
{ "value": 80, "color": "red" }
]
}
}
},
"options": {
"reduceOptions": { "calcs": ["lastNotNull"] }
}
},
{
"id": 12,
"title": "Total Lifetime Container Moves",
"type": "stat",
"description": "The total number of individual data blocks (containers) successfully relocated by the balancer.",
"gridPos": { "x": 5, "y": 21, "w": 4, "h": 6 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{
"expr": "max_over_time(disk_balancer_service_metrics_success_count{instance=~\"$datanode.*\"}[1h])",
"instant": true,
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"noValue": "0",
"unit": "short"
}
},
"options": {
"graphMode": "none",
"textMode": "value",
"reduceOptions": { "calcs": ["lastNotNull"] }
}
},
{
"id": 14,
"title": "Total Lifetime Data Balanced",
"type": "stat",
"description": "The total physical weight of all the data successfully shifted across the disks since the balancer started.",
"gridPos": { "x": 9, "y": 21, "w": 5, "h": 6 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{
"expr": "max_over_time(disk_balancer_service_metrics_success_bytes{instance=~\"$datanode.*\"}[1h])",
"instant": true,
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"noValue": "0",
"unit": "decbytes"
}
},
"options": {
"graphMode": "area",
"textMode": "auto",
"reduceOptions": { "calcs": ["lastNotNull"] }
}
},
{
"id": 15,
"title": "Failed Container Moves",
"type": "stat",
"description": "The number of times a move was aborted due to error. This should ideally stay at 0.",
"gridPos": { "x": 14, "y": 21, "w": 4, "h": 6 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{
"expr": "max_over_time(disk_balancer_service_metrics_failure_count{instance=~\"$datanode.*\"}[1h])",
"instant": true,
"refId": "A"
}
],
"fieldConfig": {
"defaults": {
"noValue": "0",
"thresholds": {
"mode": "absolute",
"steps": [
{ "value": 0, "color": "green" },
{ "value": 1, "color": "red" }
]
}
}
},
"options": {
"graphMode": "area",
"textMode": "auto",
"reduceOptions": { "calcs": ["lastNotNull"] }
}
},
{
"id": 13,
"title": "Balancer Idle Reason",
"type": "piechart",
"description": "Shows why the balancer is taking a break. 'Target Reached' means the disks are already balanced. 'Throttled' means it's slowing down on purpose so it doesn't overload the server.",
"gridPos": { "x": 18, "y": 21, "w": 6, "h": 6 },
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"targets": [
{
"expr": "sum(increase(disk_balancer_service_metrics_idle_loop_no_available_volume_pair_count[15m]))",
Comment thread
Gargi-jais11 marked this conversation as resolved.
Outdated
"instant": true,
Comment on lines +878 to +879

@Gargi-jais11 Gargi-jais11 Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
"expr": "sum(increase(disk_balancer_service_metrics_idle_loop_no_available_volume_pair_count{instance=~\"$datanode\"}[15m]))",
"instant": true,
"expr": "sum(increase(disk_balancer_service_metrics_idle_loop_no_available_volume_pair_count{instance=~\"$datanode\"}[$__range]))",
"instant": false,

"legendFormat": "Target Reached",
"refId": "A"
},
Comment thread
Gargi-jais11 marked this conversation as resolved.
{
"expr": "sum(increase(disk_balancer_service_metrics_idle_loop_exceeds_bandwidth_count[15m]))",
"instant": true,
"legendFormat": "Throttled",
"refId": "B"
}
],
"options": {
"pieType": "donut",
"legend": { "displayMode": "list", "placement": "bottom", "showLegend": true },
"reduceOptions": { "calcs": ["lastNotNull"] }
}
}
],

@Gargi-jais11 Gargi-jais11 Jul 22, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There is one more metrics which shows how many times a balancer is iterating through the loop if balancer is running.
Roughly I created this locally u can change it accordingly.

{
      "datasource": {
        "type": "prometheus",
        "uid": "${datasource}"
      },
      "description": "Rate of disk balancer scheduler loop iterations per second. A non-zero value confirms the balancer process is alive and iterating. If this drops to zero while the balancer is expected to run, the service may have stalled or been stopped.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisColorMode": "text",
            "axisLabel": "loops/sec",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "barWidthFactor": 0.6,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "insertNulls": false,
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "showValues": false,
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": 0
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 5,
        "w": 24,
        "x": 0,
        "y": 20
      },
      "id": 16,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom",
          "showLegend": true
        },
        "tooltip": {
          "hideZeros": false,
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "13.0.2",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "${datasource}"
          },
          "expr": "rate(disk_balancer_service_metrics_running_loop_count{instance=~\"$datanode\"}[$__rate_interval])",
          "legendFormat": "{{instance}} loop rate",
          "refId": "A"
        }
      ],
      "title": "Balancer Heartbeat \u2014 Scheduler Loop Rate",
      "type": "timeseries"
    }

"refresh": "5s",
"schemaVersion": 38,
"style": "dark",
"tags": ["Ozone", "DiskBalancer"],
"templating": {
"list": [
{
"current": {},
"hide": 0,
"includeAll": false,
"label": "Data Source",
"multi": false,
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"skipUrlSync": false,
"type": "datasource"
},
{
"current": {},
"datasource": { "type": "prometheus", "uid": "${datasource}" },
"definition": "label_values(disk_balancer_service_metrics_volume_data_density, instance)",
Comment thread
Gargi-jais11 marked this conversation as resolved.
Outdated
"hide": 0,
"includeAll": true,
"multi": true,
"name": "datanode",
"options": [],
"query": {
"query": "label_values(disk_balancer_service_metrics_volume_data_density, instance)",
"refId": "StandardVariableQuery"
},
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"type": "query"
}
]
},
"time": {
"from": "now-5m",
"to": "now"
},
"timezone": "browser",
"title": "Ozone Disk Balancer Operations Matrix",
Comment thread
Gargi-jais11 marked this conversation as resolved.
Outdated
"uid": "ozone-disk-balancer"
}