{ "annotations": { "list": [] }, "editable": true, "gnetId": null, "hideControls": true, "id": null, "links": [ { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [ "OS" ], "targetBlank": true, "title": "OS", "type": "dashboards" }, { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [ "MySQL" ], "targetBlank": true, "title": "MySQL", "type": "dashboards" }, { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [ "MongoDB" ], "targetBlank": true, "title": "MongoDB", "type": "dashboards" }, { "asDropdown": true, "icon": "external link", "includeVars": true, "keepTime": true, "tags": [ "HA" ], "targetBlank": true, "title": "HA", "type": "dashboards" } ], "refresh": false, "rows": [ { "collapse": false, "height": "250px", "panels": [ { "content": "You can click on an individual disk device on the legend to filter on it or multiple ones by holding Alt button.", "datasource": "Prometheus", "editable": true, "error": false, "height": "50px", "id": 8, "links": [], "mode": "text", "span": 12, "style": {}, "title": "", "transparent": true, "type": "text" }, { "aliasColors": {}, "bars": false, "datasource": "Prometheus", "decimals": 2, "editable": true, "error": false, "fill": 2, "grid": {}, "id": 11, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": true, "rightSide": true, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": false, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "span": 12, "stack": false, "steppedLine": false, "targets": [ { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "(rate(node_disk_read_time_ms{device=~\"$device\", instance=\"$host\"}[$interval]) / rate(node_disk_reads_completed{device=~\"$device\", instance=\"$host\"}[$interval])) or (irate(node_disk_read_time_ms{device=~\"$device\", instance=\"$host\"}[5m]) / irate(node_disk_reads_completed{device=~\"$device\", instance=\"$host\"}[5m]))", "interval": "$interval", "intervalFactor": 1, "legendFormat": "Read: {{ device }}", "metric": "", "refId": "A", "step": 300, "target": "" }, { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "(rate(node_disk_write_time_ms{device=~\"$device\", instance=\"$host\"}[$interval]) / rate(node_disk_writes_completed{device=~\"$device\", instance=\"$host\"}[$interval])) or (irate(node_disk_write_time_ms{device=~\"$device\", instance=\"$host\"}[5m]) / irate(node_disk_writes_completed{device=~\"$device\", instance=\"$host\"}[5m]))", "interval": "$interval", "intervalFactor": 1, "legendFormat": "Write: {{ device }}", "metric": "", "refId": "B", "step": 300, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk Latency", "description": "Shows average latency for Reads and Writes IO Devices. Higher than typical latency for highly loaded storage indicates saturation (overload) and is frequent cause of performance problems. Higher than normal latency also can indicate internal storage problems.", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ms", "label": "", "logBase": 2, "max": null, "min": 0, "show": true }, { "format": "ms", "label": "", "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "Prometheus", "decimals": 2, "editable": true, "error": false, "fill": 2, "grid": {}, "id": 15, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": false, "hideZero": true, "max": true, "min": true, "rightSide": true, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 1, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 12, "stack": false, "steppedLine": false, "targets": [ { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "rate(node_disk_reads_completed{device=~\"$device\", instance=\"$host\"}[$interval]) or irate(node_disk_reads_completed{device=~\"$device\", instance=\"$host\"}[5m])", "interval": "$interval", "intervalFactor": 1, "legendFormat": "Read: {{ device }}", "metric": "", "refId": "A", "step": 300, "target": "" }, { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "rate(node_disk_writes_completed{device=~\"$device\", instance=\"$host\"}[$interval]) or irate(node_disk_writes_completed{device=~\"$device\", instance=\"$host\"}[5m])", "interval": "$interval", "intervalFactor": 1, "legendFormat": "Write: {{ device }}", "metric": "", "refId": "B", "step": 300, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk Operations", "description": "Shows amount of physical IOs (reads and writes) different devices are serving. Spikes in number of IOs served often corresponds to performance problems due to IO subsystem overload.", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "iops", "label": "", "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "Prometheus", "decimals": 2, "editable": true, "error": false, "fill": 2, "grid": {}, "id": 16, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": false, "hideZero": true, "max": true, "min": true, "rightSide": true, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 1, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 12, "stack": false, "steppedLine": false, "targets": [ { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "rate(node_disk_bytes_read{device=~\"$device\", instance=\"$host\"}[$interval]) or irate(node_disk_bytes_read{device=~\"$device\", instance=\"$host\"}[5m])", "interval": "$interval", "intervalFactor": 1, "legendFormat": "Read: {{ device }}", "metric": "", "refId": "A", "step": 300, "target": "" }, { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "rate(node_disk_bytes_written{device=~\"$device\", instance=\"$host\"}[$interval]) or irate(node_disk_bytes_written{device=~\"$device\", instance=\"$host\"}[5m])", "interval": "$interval", "intervalFactor": 1, "legendFormat": "Write: {{ device }}", "metric": "", "refId": "B", "step": 300, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk Bandwidth", "description": "Shows volume of reads and writes the storage is handling. This can be better measure of IO capacity usage for network attached and SSD storage as it is often bandwidth limited. Amount of data being written to the disk can be used to estimate Flash storage life time.", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": "", "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "Prometheus", "decimals": 2, "editable": true, "error": false, "fill": 2, "grid": {}, "id": 14, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": false, "hideZero": true, "max": true, "min": true, "rightSide": true, "show": true, "total": false, "values": true }, "lines": false, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "span": 12, "stack": false, "steppedLine": false, "targets": [ { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "rate(node_disk_read_time_ms{device=~\"$device\", instance=\"$host\"}[$interval])/1000 or irate(node_disk_read_time_ms{device=~\"$device\", instance=\"$host\"}[5m])/1000", "interval": "$interval", "intervalFactor": 1, "legendFormat": "Read: {{ device }}", "metric": "", "refId": "A", "step": 300, "target": "" }, { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "rate(node_disk_write_time_ms{device=~\"$device\", instance=\"$host\"}[$interval])/1000 or irate(node_disk_write_time_ms{device=~\"$device\", instance=\"$host\"}[5m])/1000", "interval": "$interval", "intervalFactor": 1, "legendFormat": "Write: {{ device }}", "metric": "", "refId": "B", "step": 300, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk Load", "description": "Shows how much disk was loaded for reads or writes as average number of outstanding requests at different period of time. High disk load is a good measure of actual storage utilization. Different storage types handle load differently - some will show latency increases on low loads others can handle higher load with no problems.", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "", "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "Prometheus", "decimals": 2, "editable": true, "error": false, "fill": 2, "grid": {}, "id": 17, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": false, "hideZero": true, "max": true, "min": true, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 1, "points": false, "renderer": "flot", "seriesOverrides": [], "span": 12, "stack": false, "steppedLine": false, "targets": [ { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "rate(node_disk_io_time_ms{device=~\"$device\", instance=\"$host\"}[$interval])/1000 or irate(node_disk_io_time_ms{device=~\"$device\", instance=\"$host\"}[5m])/1000", "interval": "$interval", "intervalFactor": 1, "legendFormat": "{{ device }}", "metric": "", "refId": "A", "step": 300, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk IO Utilization", "description": "Shows disk Utilization as percent of the time when there was at least one IO request in flight. It is designed to match utilization available in iostat tool. It is not very good measure of true IO Capacity Utilization. Consider looking at IO latency and Disk Load Graphs instead.", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": "", "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": {}, "bars": false, "datasource": "Prometheus", "decimals": 2, "editable": true, "error": false, "fill": 2, "grid": {}, "id": 18, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": true, "rightSide": true, "show": true, "total": false, "values": true }, "lines": false, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "span": 12, "stack": false, "steppedLine": false, "targets": [ { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "(1 + rate(node_disk_reads_merged{device=~\"$device\", instance=\"$host\"}[$interval]) / rate(node_disk_reads_completed{device=~\"$device\", instance=\"$host\"}[$interval])) or (1 + irate(node_disk_reads_merged{device=~\"$device\", instance=\"$host\"}[5m]) / irate(node_disk_reads_completed{device=~\"$device\", instance=\"$host\"}[5m]))", "interval": "$interval", "intervalFactor": 1, "legendFormat": "Read Ratio: {{ device }}", "metric": "", "refId": "A", "step": 300, "target": "" }, { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "(1 + rate(node_disk_writes_merged{device=~\"$device\", instance=\"$host\"}[$interval]) / rate(node_disk_writes_completed{device=~\"$device\", instance=\"$host\"}[$interval])) or (1 + irate(node_disk_writes_merged{device=~\"$device\", instance=\"$host\"}[5m]) / irate(node_disk_writes_completed{device=~\"$device\", instance=\"$host\"}[5m]))", "interval": "$interval", "intervalFactor": 1, "legendFormat": "Write Ratio: {{ device }}", "metric": "", "refId": "B", "step": 300, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk Operations Merge Ratio", "description": "Shows how effectively Operating System is able to merge logical IO requests into physical requests. This is a good measure of the IO locality which can be used for workload characterization.", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "", "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": 0, "show": true } ] }, { "aliasColors": { "Read IO size: sdb": "#2F575E", "Read: sdb": "#3F6833" }, "bars": false, "datasource": "Prometheus", "decimals": 2, "editable": true, "error": false, "fill": 2, "grid": {}, "id": 20, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": true, "rightSide": true, "show": true, "total": false, "values": true }, "lines": false, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "span": 12, "stack": false, "steppedLine": false, "targets": [ { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "rate(node_disk_sectors_read{instance=\"$host\", device=~\"$device\"}[$interval]) * 512 / rate(node_disk_reads_completed{instance=\"$host\", device=~\"$device\"}[$interval]) or irate(node_disk_sectors_read{instance=\"$host\", device=~\"$device\"}[5m]) * 512 / irate(node_disk_reads_completed{instance=\"$host\", device=~\"$device\"}[5m]) ", "interval": "$interval", "intervalFactor": 1, "legendFormat": "Read size: {{ device }}", "metric": "", "refId": "A", "step": 300, "target": "" }, { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "rate(node_disk_sectors_written{instance=\"$host\", device=~\"$device\"}[$interval]) * 512 / rate(node_disk_writes_completed{instance=\"$host\", device=~\"$device\"}[$interval]) or irate(node_disk_sectors_written{instance=\"$host\", device=~\"$device\"}[5m]) * 512 / irate(node_disk_writes_completed{instance=\"$host\", device=~\"$device\"}[5m]) ", "interval": "$interval", "intervalFactor": 1, "legendFormat": "Write size: {{ device }}", "metric": "", "refId": "B", "step": 300, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk IO Size", "description": "Shows average size of a single disk operation.", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": "", "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": 0, "show": true } ] } ], "repeat": null, "repeatIteration": null, "repeatRowId": null, "showTitle": false, "title": "Disk Stats", "titleSize": "h6" } ], "schemaVersion": 13, "sharedCrosshair": true, "style": "dark", "tags": [ "OS", "Percona" ], "templating": { "list": [ { "allFormat": "glob", "auto": true, "auto_count": 200, "auto_min": "1s", "current": { "text": "auto", "value": "$__auto_interval" }, "datasource": "Prometheus", "hide": 0, "includeAll": false, "label": "Interval", "multi": false, "multiFormat": "glob", "name": "interval", "options": [ { "selected": true, "text": "auto", "value": "$__auto_interval" }, { "selected": false, "text": "1s", "value": "1s" }, { "selected": false, "text": "5s", "value": "5s" }, { "selected": false, "text": "1m", "value": "1m" }, { "selected": false, "text": "5m", "value": "5m" }, { "selected": false, "text": "1h", "value": "1h" }, { "selected": false, "text": "6h", "value": "6h" }, { "selected": false, "text": "1d", "value": "1d" } ], "query": "1s,5s,1m,5m,1h,6h,1d", "refresh": 2, "type": "interval" }, { "allFormat": "glob", "allValue": null, "datasource": "Prometheus", "hide": 0, "includeAll": false, "label": "Host", "multi": false, "multiFormat": "regex values", "name": "host", "query": "label_values(node_disk_reads_completed, instance)", "refresh": 1, "refresh_on_load": false, "regex": "", "sort": 1, "tagValuesQuery": "instance", "tagsQuery": "up", "type": "query", "useTags": false }, { "allFormat": "glob", "allValue": null, "datasource": "Prometheus", "hide": 0, "includeAll": true, "label": "Device", "multi": true, "multiFormat": "regex values", "name": "device", "query": "label_values(node_disk_reads_completed{instance=\"$host\", device!~\"dm-.+\"}, device)", "refresh": 1, "refresh_on_load": false, "regex": "", "sort": 1, "tagValuesQuery": "instance", "tagsQuery": "up", "type": "query", "useTags": false } ] }, "time": { "from": "now-12h", "to": "now" }, "timepicker": { "collapse": false, "enable": true, "notice": false, "now": true, "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "status": "Stable", "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ], "type": "timepicker" }, "timezone": "browser", "title": "Disk Performance", "version": 0 }