{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__elements": {}, "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "11.2.0" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, { "type": "panel", "id": "table", "name": "Table", "version": "" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, "links": [], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 100, "panels": [], "title": "Migration Health Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Time since the most recent lifecycle cycle completed across all source pods/groups. Daily-batch SLA: red if older than 26h.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 93600 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, "id": 101, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "time() - max(banyandb_lifecycle_last_run_timestamp_seconds{job=~\"$job\", group=~\"$group\"})", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Last Migration (age)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Lifecycle migration cycles completed in the last 24h (sum of increase over banyandb_lifecycle_cycles_total).", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, "id": 102, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "sum(increase(banyandb_lifecycle_cycles_total{job=~\"$job\", group=~\"$group\"}[24h]))", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Cycles (24h)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Number of source pods whose most recent migration cycle reported failure (last_run_success==0). Red if any.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, "id": 103, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "count(banyandb_lifecycle_last_run_success{job=~\"$job\", group=~\"$group\"} == 0) or vector(0)", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Last-run Failures", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total bytes sent by the lifecycle publisher in the last 24h (file-sync sent_bytes).", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, "id": 104, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "sum(increase(banyandb_lifecycle_migration_sent_bytes{job=~\"$job\", group=~\"$group\"}[24h]))", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Data Migrated (24h)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Total lifecycle migration errors in the last 24h. Red if any.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, "id": 105, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "sum(increase(banyandb_lifecycle_migration_total_err{job=~\"$job\"}[24h])) or vector(0)", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Migration Errors (24h)", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Distinct source pods currently emitting lifecycle migration metrics.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, "id": 106, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "count(count by (pod_name)(banyandb_lifecycle_migration_total_finished{job=~\"$job\"}))", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Active Source Pods", "type": "stat" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 110, "panels": [], "title": "Cycle Status", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Completed migration cycles per source pod and group (banyandb_lifecycle_cycles_total). This counter is attributed to the SOURCE node only \u2014 its remote_* labels carry the lifecycle node's own (sender) identity, not the migration destination, so there is no destination column here. A single cycle may fan out to multiple destination tiers; see the Flows table for the per-flow source\u2192dest breakdown.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "id": 111, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": [ "sum" ], "countRows": false, "fields": "" } }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "sum by (node_type, pod_name, group)(banyandb_lifecycle_cycles_total{job=~\"$job\", group=~\"$group\"})", "format": "table", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A" } ], "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "job": true, "instance": true, "pod": true, "container_name": true, "node_role": true, "remote_role": true, "remote_tier": true, "remote_node": true, "__name__": true }, "renameByName": { "node_type": "Src tier", "pod_name": "Src pod", "group": "Group", "Value": "Cycles" }, "indexByName": { "node_type": 0, "pod_name": 1, "group": 2, "Value": 3 } } } ], "title": "Cycle Ledger (per source pod \u00d7 group)", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Per source pod: age of the last completed cycle and its success flag.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "inspect": false }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Last run age" }, "properties": [ { "id": "unit", "value": "s" } ] }, { "matcher": { "id": "byName", "options": "Success" }, "properties": [ { "id": "mappings", "value": [ { "type": "value", "options": { "1": { "text": "OK", "color": "green", "index": 0 }, "0": { "text": "FAIL", "color": "red", "index": 1 } } } ] }, { "id": "custom.cellOptions", "value": { "type": "color-background", "mode": "basic" } } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "id": 112, "options": { "showHeader": true, "cellHeight": "sm", "footer": { "show": false, "reducer": [ "sum" ], "countRows": false, "fields": "" } }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "time() - banyandb_lifecycle_last_run_timestamp_seconds{job=~\"$job\", group=~\"$group\"}", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A", "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "max by (pod_name)(banyandb_lifecycle_last_run_success{job=~\"$job\", group=~\"$group\"})", "instant": true, "legendFormat": "__auto", "range": false, "refId": "B", "format": "table" } ], "transformations": [ { "id": "joinByField", "options": { "byField": "pod_name", "mode": "outer" } }, { "id": "organize", "options": { "excludeByName": { "Time": true, "job": true, "instance": true, "pod": true, "container_name": true, "node_role": true, "remote_role": true, "remote_node": true, "remote_tier": true, "group": true, "__name__": true }, "renameByName": { "pod_name": "Src pod", "node_type": "Tier", "Value #A": "Last run age", "Value #B": "Success" }, "indexByName": { "pod_name": 0, "node_type": 1, "Value #A": 2, "Value #B": 3 } } } ], "title": "Last Run per Source Pod", "type": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Completed migration cycles per source tier over each interval (banyandb_lifecycle_cycles_total). Attributed to the source node only \u2014 cycles_total has no destination dimension.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 14 }, "id": 113, "options": { "legend": { "calcs": [ "lastNotNull", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (node_type)(increase(banyandb_lifecycle_cycles_total{job=~\"$job\", group=~\"$group\"}[$__interval]))", "instant": false, "legendFormat": "{{node_type}} tier", "range": true, "refId": "A" } ], "title": "Cycles Over Time (by source tier)", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, "id": 120, "panels": [], "title": "Migration Flows", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Cross-tier lifecycle migration flows (hot\u2192warm, warm\u2192cold). Each row is one directed source\u2192target per (group, operation). Pub side is the lifecycle sidecar inside the SOURCE data pod (publishes via banyandb_lifecycle_migration_*); Sub side is the data pod RECEIVING (records via banyandb_queue_sub_total_message_started for batch-write, banyandb_queue_sub_total_finished for file-sync, and total_batch_started{remote_role=\"lifecycle\"}). Counts/bytes/errors are per-day (rate \u00d7 86400) because tier migration is a daily-batch workload and an instant per-second rate is structurally zero 99% of the time. Pub msg/day and Sub msg/day should match side-to-side for BOTH operations: batch-write replay rows (operation=\"batch-write\") pair Pub total_finished against Sub total_message_started (per-message), while file-sync rows (operation=\"file-sync\", chunked_sync RPC) pair Pub total_finished against Sub total_finished (per-part) \u2014 the chunked-sync receive path has no per-message counter, so Sub msg/day for file-sync counts synced parts, not messages. Pub batch/day and Sub batch/day compare per-batch counts and only apply to the batch-write rows. Latencies are p99 over the visible time range, NOT per-day \u2014 they are time-domain quantiles, not rates.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "cellOptions": { "type": "auto" }, "filterable": false, "inspect": false }, "decimals": 2, "mappings": [ { "type": "special", "options": { "match": "nan", "result": { "text": "-", "index": 0 } } } ], "noValue": "-", "thresholds": { "mode": "absolute", "steps": [ { "color": "text", "value": null } ] }, "unit": "short", "overrides": [ { "matcher": { "id": "byName", "options": "Pub msg/day" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 } ] }, { "matcher": { "id": "byName", "options": "Sub msg/day" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 } ] }, { "matcher": { "id": "byName", "options": "Pub err/day" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "text", "value": null }, { "color": "red", "value": 1e-06 } ] } }, { "id": "custom.cellOptions", "value": { "type": "color-text" } } ] }, { "matcher": { "id": "byName", "options": "Sub err/day" }, "properties": [ { "id": "unit", "value": "short" }, { "id": "decimals", "value": 2 }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "text", "value": null }, { "color": "red", "value": 1e-06 } ] } }, { "id": "custom.cellOptions", "value": { "type": "color-text" } } ] }, { "matcher": { "id": "byName", "options": "Pub B/day" }, "properties": [ { "id": "unit", "value": "bytes" } ] }, { "matcher": { "id": "byName", "options": "Sub B/day" }, "properties": [ { "id": "unit", "value": "bytes" } ] } ] }, "overrides": [ { "matcher": { "id": "byName", "options": "Pub p99" }, "properties": [ { "id": "unit", "value": "s" }, { "id": "decimals", "value": 3 } ] }, { "matcher": { "id": "byName", "options": "Sub p99" }, "properties": [ { "id": "unit", "value": "s" }, { "id": "decimals", "value": 3 } ] }, { "matcher": { "id": "byName", "options": "Pub B/s" }, "properties": [ { "id": "unit", "value": "Bps" } ] }, { "matcher": { "id": "byName", "options": "Sub B/s" }, "properties": [ { "id": "unit", "value": "Bps" } ] }, { "matcher": { "id": "byName", "options": "Pub err/s" }, "properties": [ { "id": "unit", "value": "ops" }, { "id": "custom.cellOptions", "value": { "type": "color-text" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "text", "value": null }, { "color": "red", "value": 1e-06 } ] } } ] }, { "matcher": { "id": "byName", "options": "Sub err/s" }, "properties": [ { "id": "unit", "value": "ops" }, { "id": "custom.cellOptions", "value": { "type": "color-text" } }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "text", "value": null }, { "color": "red", "value": 1e-06 } ] } } ] }, { "matcher": { "id": "byName", "options": "Source" }, "properties": [ { "id": "custom.filterable", "value": true } ] }, { "matcher": { "id": "byName", "options": "Target" }, "properties": [ { "id": "custom.filterable", "value": true } ] }, { "matcher": { "id": "byName", "options": "Operation" }, "properties": [ { "id": "custom.filterable", "value": true } ] } ] }, "gridPos": { "h": 13, "w": 24, "x": 0, "y": 22 }, "id": 121, "options": { "cellHeight": "sm", "footer": { "countRows": false, "fields": "", "reducer": [ "sum" ], "show": false }, "showHeader": true, "sortBy": [ { "desc": true, "displayName": "Pub msg/day" } ] }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "sum by (source, target, operation) (label_replace(label_replace(rate(banyandb_lifecycle_migration_total_finished{job=~\"$job\", group=~\"$group\", operation=~\"$operation\"}[$__range]) *86400 , \"source\", \"$1\", \"pod_name\", \"(.*)\"), \"target\", \"$1\", \"remote_node\", \"([^.:]+).*\"))", "instant": true, "legendFormat": "__auto", "range": false, "refId": "A", "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "sum by (source, target, operation) (label_replace(label_replace(rate(banyandb_queue_sub_total_message_started{job=~\"$job\", group=~\"$group\", operation=~\"$operation\", remote_role=\"lifecycle\"}[$__range]) *86400 or rate(banyandb_queue_sub_total_finished{job=~\"$job\", group=~\"$group\", remote_role=\"lifecycle\", operation=~\"$operation\", operation=\"file-sync\"}[$__range]) *86400, \"source\", \"$1\", \"remote_node\", \"([^.:]+).*\"), \"target\", \"$1\", \"pod_name\", \"(.*)\"))", "instant": true, "legendFormat": "__auto", "range": false, "refId": "B", "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "histogram_quantile(0.99, sum by (le, source, target, operation) (label_replace(label_replace(rate(banyandb_lifecycle_migration_total_latency_bucket{job=~\"$job\", group=~\"$group\", operation=~\"$operation\"}[$__range]), \"source\", \"$1\", \"pod_name\", \"(.*)\"), \"target\", \"$1\", \"remote_node\", \"([^.:]+).*\")))", "instant": true, "legendFormat": "__auto", "range": false, "refId": "C", "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "histogram_quantile(0.99, sum by (le, source, target, operation) (label_replace(label_replace(rate(banyandb_queue_sub_total_latency_bucket{job=~\"$job\", group=~\"$group\", operation=~\"$operation\", remote_role=\"lifecycle\"}[$__range]), \"source\", \"$1\", \"remote_node\", \"([^.:]+).*\"), \"target\", \"$1\", \"pod_name\", \"(.*)\")))", "instant": true, "legendFormat": "__auto", "range": false, "refId": "D", "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "sum by (source, target, operation) (label_replace(label_replace(rate(banyandb_lifecycle_migration_total_err{job=~\"$job\", group=~\"$group\", operation=~\"$operation\"}[$__range]) *86400 , \"source\", \"$1\", \"pod_name\", \"(.*)\"), \"target\", \"$1\", \"remote_node\", \"([^.:]+).*\"))", "instant": true, "legendFormat": "__auto", "range": false, "refId": "E", "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "sum by (source, target, operation) (label_replace(label_replace(rate(banyandb_queue_sub_total_err{job=~\"$job\", group=~\"$group\", operation=~\"$operation\", remote_role=\"lifecycle\"}[$__range]) *86400 , \"source\", \"$1\", \"remote_node\", \"([^.:]+).*\"), \"target\", \"$1\", \"pod_name\", \"(.*)\"))", "instant": true, "legendFormat": "__auto", "range": false, "refId": "F", "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "sum by (source, target, operation) (label_replace(label_replace(rate(banyandb_lifecycle_migration_sent_bytes{job=~\"$job\", group=~\"$group\", operation=~\"$operation\"}[$__range]) *86400 , \"source\", \"$1\", \"pod_name\", \"(.*)\"), \"target\", \"$1\", \"remote_node\", \"([^.:]+).*\"))", "instant": true, "legendFormat": "__auto", "range": false, "refId": "G", "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "sum by (source, target, operation) (label_replace(label_replace(rate(banyandb_queue_sub_received_bytes{job=~\"$job\", group=~\"$group\", operation=~\"$operation\", remote_role=\"lifecycle\"}[$__range]) *86400 , \"source\", \"$1\", \"remote_node\", \"([^.:]+).*\"), \"target\", \"$1\", \"pod_name\", \"(.*)\"))", "instant": true, "legendFormat": "__auto", "range": false, "refId": "H", "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "sum by (source, target, operation) (label_replace(label_replace(rate(banyandb_lifecycle_migration_total_batch_finished{job=~\"$job\", group=~\"$group\", operation=~\"$operation\"}[$__range]) *86400 , \"source\", \"$1\", \"pod_name\", \"(.*)\"), \"target\", \"$1\", \"remote_node\", \"([^.:]+).*\"))", "instant": true, "legendFormat": "__auto", "range": false, "refId": "I", "format": "table" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "sum by (source, target, operation) (label_replace(label_replace(rate(banyandb_queue_sub_total_batch_started{job=~\"$job\", group=~\"$group\", operation=~\"$operation\", remote_role=\"lifecycle\"}[$__range]) *86400 , \"source\", \"$1\", \"remote_node\", \"([^.:]+).*\"), \"target\", \"$1\", \"pod_name\", \"(.*)\"))", "instant": true, "legendFormat": "__auto", "range": false, "refId": "J", "format": "table" } ], "title": "Migration Flows \u2014 Pub vs Sub (per flow)", "transformations": [ { "id": "merge", "options": {} }, { "id": "organize", "options": { "excludeByName": { "Time": true }, "indexByName": { "source": 0, "target": 1, "operation": 2, "Value #A": 3, "Value #B": 4, "Value #C": 5, "Value #D": 6, "Value #E": 7, "Value #F": 8, "Value #G": 9, "Value #H": 10, "Value #I": 11, "Value #J": 12 }, "renameByName": { "source": "Source", "target": "Target", "operation": "Operation", "Value #A": "Pub msg/day", "Value #B": "Sub msg/day", "Value #C": "Pub p99", "Value #D": "Sub p99", "Value #E": "Pub err/day", "Value #F": "Sub err/day", "Value #G": "Pub B/day", "Value #H": "Sub B/day", "Value #I": "Pub batch/day", "Value #J": "Sub batch/day" } } } ], "type": "table" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 35 }, "id": 130, "panels": [], "title": "Throughput & Volume", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Publisher sent-bytes rate per flow and operation.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 36 }, "id": 131, "options": { "legend": { "calcs": [ "lastNotNull", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (node_type, remote_tier, operation)(rate(banyandb_lifecycle_migration_sent_bytes{job=~\"$job\", group=~\"$group\", operation=~\"$operation\"}[$__rate_interval]))", "instant": false, "legendFormat": "{{node_type}}\u2192{{remote_tier}} {{operation}}", "range": true, "refId": "A" } ], "title": "Bytes/s by Flow", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Per-message replay rate for batch-write migration.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 36 }, "id": 132, "options": { "legend": { "calcs": [ "lastNotNull", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (node_type, remote_tier)(rate(banyandb_lifecycle_migration_total_finished{job=~\"$job\", group=~\"$group\", operation=\"batch-write\"}[$__rate_interval]))", "instant": false, "legendFormat": "{{node_type}}\u2192{{remote_tier}}", "range": true, "refId": "A" } ], "title": "Docs Replayed/s (batch-write)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Per-part chunked-sync completion rate for file-sync migration.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 36 }, "id": 133, "options": { "legend": { "calcs": [ "lastNotNull", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (node_type, remote_tier)(rate(banyandb_lifecycle_migration_total_finished{job=~\"$job\", group=~\"$group\", operation=\"file-sync\"}[$__rate_interval]))", "instant": false, "legendFormat": "{{node_type}}\u2192{{remote_tier}}", "range": true, "refId": "A" } ], "title": "Parts Synced/s (file-sync)", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 44 }, "id": 140, "panels": [], "title": "Latency", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "p99 of banyandb_lifecycle_migration_total_latency by operation and flow.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 45 }, "id": 141, "options": { "legend": { "calcs": [ "lastNotNull", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le, operation, node_type, remote_tier)(rate(banyandb_lifecycle_migration_total_latency_bucket{job=~\"$job\", group=~\"$group\", operation=~\"$operation\"}[$__rate_interval])))", "instant": false, "legendFormat": "{{node_type}}\u2192{{remote_tier}} {{operation}}", "range": true, "refId": "A" } ], "title": "p99 Migration Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "p99 of banyandb_lifecycle_migration_total_batch_latency per flow.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 45 }, "id": 142, "options": { "legend": { "calcs": [ "lastNotNull", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum by (le, node_type, remote_tier)(rate(banyandb_lifecycle_migration_total_batch_latency_bucket{job=~\"$job\", group=~\"$group\"}[$__rate_interval])))", "instant": false, "legendFormat": "{{node_type}}\u2192{{remote_tier}}", "range": true, "refId": "A" } ], "title": "p99 Batch Latency (batch-write)", "type": "timeseries" }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 53 }, "id": 150, "panels": [], "title": "Errors & Integrity", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Lifecycle migration error rate by error_type and operation. Empty when there are no errors.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 1 } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 54 }, "id": 151, "options": { "legend": { "calcs": [ "lastNotNull", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (error_type, operation)(rate(banyandb_lifecycle_migration_total_err{job=~\"$job\", group=~\"$group\"}[$__rate_interval]))", "instant": false, "legendFormat": "{{operation}} {{error_type}}", "range": true, "refId": "A" } ], "title": "Errors/s by Type", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Publisher-finished minus subscriber-received per operation; should hover at 0. Non-zero means a side dropped or double-counted.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "barWidthFactor": 0.6, "drawStyle": "line", "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 54 }, "id": 152, "options": { "legend": { "calcs": [ "lastNotNull", "max", "mean" ], "displayMode": "table", "placement": "bottom", "showLegend": true, "sortBy": "Last *", "sortDesc": true }, "tooltip": { "mode": "multi", "sort": "desc" } }, "pluginVersion": "11.2.0", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(rate(banyandb_lifecycle_migration_total_finished{job=~\"$job\", group=~\"$group\", operation=\"batch-write\"}[$__rate_interval])) - sum(rate(banyandb_queue_sub_total_message_started{job=~\"$job\", group=~\"$group\", remote_role=\"lifecycle\", operation=\"batch-write\"}[$__rate_interval]))", "instant": false, "legendFormat": "batch-write drift", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(rate(banyandb_lifecycle_migration_total_finished{job=~\"$job\", group=~\"$group\", operation=\"file-sync\"}[$__rate_interval])) - sum(rate(banyandb_queue_sub_total_finished{job=~\"$job\", group=~\"$group\", remote_role=\"lifecycle\", operation=\"file-sync\"}[$__rate_interval]))", "instant": false, "legendFormat": "file-sync drift", "range": true, "refId": "B" } ], "title": "Pub\u2194Sub Drift", "type": "timeseries" } ], "refresh": "5m", "schemaVersion": 39, "tags": [ "banyandb", "lifecycle", "migration", "fodc" ], "templating": { "list": [ { "current": {}, "hide": 0, "includeAll": false, "label": "Prometheus", "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "type": "datasource" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(banyandb_lifecycle_migration_total_finished,job)", "hide": 0, "includeAll": false, "label": "job", "name": "job", "options": [], "query": { "qryType": 1, "query": "label_values(banyandb_lifecycle_migration_total_finished,job)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", "sort": 1, "type": "query" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(banyandb_lifecycle_migration_total_finished{job=~\"$job\"},group)", "hide": 0, "includeAll": true, "allValue": ".*", "label": "group", "name": "group", "multi": true, "options": [], "query": { "qryType": 1, "query": "label_values(banyandb_lifecycle_migration_total_finished{job=~\"$job\"},group)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", "sort": 1, "type": "query" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(banyandb_lifecycle_migration_total_finished{job=~\"$job\"},operation)", "hide": 0, "includeAll": true, "allValue": ".*", "label": "operation", "name": "operation", "multi": true, "options": [], "query": { "qryType": 1, "query": "label_values(banyandb_lifecycle_migration_total_finished{job=~\"$job\"},operation)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 2, "regex": "", "sort": 1, "type": "query" } ] }, "time": { "from": "now-2d", "to": "now" }, "timepicker": {}, "timezone": "", "title": "BanyanDB \u2014 Lifecycle Migration", "uid": "banyandb-fodc-migration", "version": 1, "weekStart": "" }