dm/container/conf/tool/grafana/dashboards/Prometheus.json

{
    "annotations": {
        "list": []
    },
    "editable": true,
    "gnetId": null,
    "graphTooltip": 1,
    "hideControls": true,
    "id": null,
    "links": [],
    "refresh": false,
    "rows": [
        {
            "collapse": false,
            "height": "125px",
            "panels": [
                {
                    "cacheTimeout": null,
                    "colorBackground": false,
                    "colorValue": false,
                    "colors": [
                        "rgba(245, 54, 54, 0.9)",
                        "rgba(237, 129, 40, 0.89)",
                        "rgba(50, 172, 45, 0.97)"
                    ],
                    "datasource": "Prometheus",
                    "editable": true,
                    "error": false,
                    "format": "none",
                    "gauge": {
                        "maxValue": 100,
                        "minValue": 0,
                        "show": false,
                        "thresholdLabels": false,
                        "thresholdMarkers": true
                    },
                    "height": "125px",
                    "hideTimeOverride": false,
                    "id": 8,
                    "interval": "$interval",
                    "links": [],
                    "mappingType": 1,
                    "mappingTypes": [
                        {
                            "name": "value to text",
                            "value": 1
                        },
                        {
                            "name": "range to text",
                            "value": 2
                        }
                    ],
                    "maxDataPoints": 100,
                    "nullPointMode": "connected",
                    "nullText": null,
                    "postfix": "",
                    "postfixFontSize": "50%",
                    "prefix": "",
                    "prefixFontSize": "50%",
                    "rangeMaps": [
                        {
                            "from": "null",
                            "text": "N/A",
                            "to": "null"
                        }
                    ],
                    "span": 3,
                    "sparkline": {
                        "fillColor": "rgba(31, 118, 189, 0.18)",
                        "full": false,
                        "lineColor": "rgb(31, 120, 193)",
                        "show": false
                    },
                    "tableColumn": "",
                    "targets": [
                        {
                            "expr": "prometheus_local_storage_memory_series",
                            "interval": "5m",
                            "intervalFactor": 1,
                            "refId": "A",
                            "step": 300
                        }
                    ],
                    "thresholds": "",
                    "timeFrom": null,
                    "title": "Time Series",
                    "type": "singlestat",
                    "valueFontSize": "80%",
                    "valueMaps": [
                        {
                            "op": "=",
                            "text": "N/A",
                            "value": "null"
                        }
                    ],
                    "valueName": "current"
                },
                {
                    "cacheTimeout": null,
                    "colorBackground": false,
                    "colorValue": false,
                    "colors": [
                        "rgba(245, 54, 54, 0.9)",
                        "rgba(237, 129, 40, 0.89)",
                        "rgba(50, 172, 45, 0.97)"
                    ],
                    "datasource": "Prometheus",
                    "editable": true,
                    "error": false,
                    "format": "none",
                    "gauge": {
                        "maxValue": 100,
                        "minValue": 0,
                        "show": false,
                        "thresholdLabels": false,
                        "thresholdMarkers": true
                    },
                    "height": "125px",
                    "hideTimeOverride": false,
                    "id": 9,
                    "interval": "$interval",
                    "links": [],
                    "mappingType": 1,
                    "mappingTypes": [
                        {
                            "name": "value to text",
                            "value": 1
                        },
                        {
                            "name": "range to text",
                            "value": 2
                        }
                    ],
                    "maxDataPoints": 100,
                    "nullPointMode": "connected",
                    "nullText": null,
                    "postfix": "",
                    "postfixFontSize": "50%",
                    "prefix": "",
                    "prefixFontSize": "50%",
                    "rangeMaps": [
                        {
                            "from": "null",
                            "text": "N/A",
                            "to": "null"
                        }
                    ],
                    "span": 3,
                    "sparkline": {
                        "fillColor": "rgba(31, 118, 189, 0.18)",
                        "full": false,
                        "lineColor": "rgb(31, 120, 193)",
                        "show": false
                    },
                    "tableColumn": "",
                    "targets": [
                        {
                            "expr": "prometheus_local_storage_memory_chunks",
                            "interval": "5m",
                            "intervalFactor": 1,
                            "refId": "A",
                            "step": 300
                        }
                    ],
                    "thresholds": "",
                    "timeFrom": null,
                    "title": "Memory Chunks",
                    "type": "singlestat",
                    "valueFontSize": "80%",
                    "valueMaps": [
                        {
                            "op": "=",
                            "text": "N/A",
                            "value": "null"
                        }
                    ],
                    "valueName": "current"
                },
                {
                    "cacheTimeout": null,
                    "colorBackground": false,
                    "colorValue": false,
                    "colors": [
                        "rgba(245, 54, 54, 0.9)",
                        "rgba(237, 129, 40, 0.89)",
                        "rgba(50, 172, 45, 0.97)"
                    ],
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "format": "\u00b5s",
                    "gauge": {
                        "maxValue": 100,
                        "minValue": 0,
                        "show": false,
                        "thresholdLabels": false,
                        "thresholdMarkers": true
                    },
                    "height": "125px",
                    "hideTimeOverride": true,
                    "id": 11,
                    "interval": "$interval",
                    "links": [],
                    "mappingType": 1,
                    "mappingTypes": [
                        {
                            "name": "value to text",
                            "value": 1
                        },
                        {
                            "name": "range to text",
                            "value": 2
                        }
                    ],
                    "maxDataPoints": 100,
                    "nullPointMode": "connected",
                    "nullText": null,
                    "postfix": "",
                    "postfixFontSize": "50%",
                    "prefix": "",
                    "prefixFontSize": "50%",
                    "rangeMaps": [
                        {
                            "from": "null",
                            "text": "N/A",
                            "to": "null"
                        }
                    ],
                    "span": 3,
                    "sparkline": {
                        "fillColor": "rgba(31, 118, 189, 0.18)",
                        "full": true,
                        "lineColor": "rgb(31, 120, 193)",
                        "show": true
                    },
                    "tableColumn": "",
                    "targets": [
                        {
                            "expr": "sum(rate(http_request_duration_microseconds_sum{job=\"prometheus\"}[$interval])) /  sum(rate(http_request_duration_microseconds_count{job=\"prometheus\"}[$interval])) or sum(irate(http_request_duration_microseconds_sum{job=\"prometheus\"}[5m])) /  sum(irate(http_request_duration_microseconds_count{job=\"prometheus\"}[5m]))",
                            "interval": "5m",
                            "intervalFactor": 1,
                            "legendFormat": "",
                            "refId": "A",
                            "step": 300
                        }
                    ],
                    "thresholds": "",
                    "timeFrom": "12h",
                    "timeShift": null,
                    "title": "HTTP Request Duration",
                    "type": "singlestat",
                    "valueFontSize": "80%",
                    "valueMaps": [
                        {
                            "op": "=",
                            "text": "N/A",
                            "value": "null"
                        }
                    ],
                    "valueName": "current"
                },
                {
                    "cacheTimeout": null,
                    "colorBackground": false,
                    "colorValue": false,
                    "colors": [
                        "rgba(245, 54, 54, 0.9)",
                        "rgba(237, 129, 40, 0.89)",
                        "rgba(50, 172, 45, 0.97)"
                    ],
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "format": "bytes",
                    "gauge": {
                        "maxValue": 100,
                        "minValue": 0,
                        "show": false,
                        "thresholdLabels": false,
                        "thresholdMarkers": true
                    },
                    "height": "125px",
                    "hideTimeOverride": true,
                    "id": 10,
                    "interval": "$interval",
                    "links": [],
                    "mappingType": 1,
                    "mappingTypes": [
                        {
                            "name": "value to text",
                            "value": 1
                        },
                        {
                            "name": "range to text",
                            "value": 2
                        }
                    ],
                    "maxDataPoints": 100,
                    "nullPointMode": "connected",
                    "nullText": null,
                    "postfix": "",
                    "postfixFontSize": "50%",
                    "prefix": "",
                    "prefixFontSize": "50%",
                    "rangeMaps": [
                        {
                            "from": "null",
                            "text": "N/A",
                            "to": "null"
                        }
                    ],
                    "span": 3,
                    "sparkline": {
                        "fillColor": "rgba(31, 118, 189, 0.18)",
                        "full": true,
                        "lineColor": "rgb(31, 120, 193)",
                        "show": true
                    },
                    "tableColumn": "",
                    "targets": [
                        {
                            "expr": "sum(rate(http_response_size_bytes_sum{job=\"prometheus\"}[$interval])) /  sum(rate(http_response_size_bytes_count{job=\"prometheus\"}[$interval])) or sum(irate(http_response_size_bytes_sum{job=\"prometheus\"}[5m])) /  sum(irate(http_response_size_bytes_count{job=\"prometheus\"}[5m]))",
                            "interval": "5m",
                            "intervalFactor": 1,
                            "refId": "A",
                            "step": 300
                        }
                    ],
                    "thresholds": "",
                    "timeFrom": "12h",
                    "title": "HTTP Response Size",
                    "type": "singlestat",
                    "valueFontSize": "80%",
                    "valueMaps": [
                        {
                            "op": "=",
                            "text": "N/A",
                            "value": "null"
                        }
                    ],
                    "valueName": "current"
                }
            ],
            "repeat": null,
            "repeatIteration": null,
            "repeatRowId": null,
            "showTitle": false,
            "title": "Widgetst",
            "titleSize": "h6"
        },
        {
            "collapse": false,
            "height": "250px",
            "panels": [
                {
                    "aliasColors": {
                        "Scrapes Performed": "#6ED0E0"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 2,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "show": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 2,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 5,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [
                        {
                            "alias": "Scrapes Performed",
                            "yaxis": 2
                        }
                    ],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "rate(prometheus_local_storage_ingested_samples_total[$interval]) or irate(prometheus_local_storage_ingested_samples_total[5m])",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Samples Ingested",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        },
                        {
                            "expr": "sum(rate(prometheus_target_interval_length_seconds_count[$interval])) or sum(irate(prometheus_target_interval_length_seconds_count[5m]))",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Scrapes Performed",
                            "refId": "B",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Ingestion",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "cumulative"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "short",
                            "label": "",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "ops",
                            "label": "",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839",
                        "Min Successfully Scraped Targets": "#E24D42"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 12,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 2,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 2,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [
                        {
                            "alias": "Min Successfully Scraped Targets",
                            "lines": false,
                            "pointradius": 1,
                            "points": true
                        }
                    ],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "count(up)",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Targets Configured",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        },
                        {
                            "expr": "sum(avg_over_time(up[$interval])) or  sum(avg_over_time(up[5m]))",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Avg Succesfully Scraped Targets",
                            "refId": "B",
                            "step": 60
                        },
                        {
                            "expr": "sum(min_over_time(up[$interval])) or  sum(min_over_time(up[5m]))",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Min Successfully Scraped Targets",
                            "refId": "C",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Prometheus Targets",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 15,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 2,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 2,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "prometheus_local_storage_memory_series{instance=\"prometheus\"}",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Time Series in Memory",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        },
                        {
                            "expr": "prometheus_local_storage_open_head_chunks{instance=\"prometheus\"}",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "Open Heads Chinks",
                            "refId": "B",
                            "step": 40
                        },
                        {
                            "expr": "prometheus_local_storage_memory_dirty_series{instance=\"prometheus\"}",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "Dirty Time Series",
                            "refId": "C",
                            "step": 40
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Time Series",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 17,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "total": false,
                        "values": true
                    },
                    "lines": false,
                    "linewidth": 2,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 1,
                    "points": true,
                    "renderer": "flot",
                    "seriesOverrides": [],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "rate(prometheus_local_storage_persist_errors_total[$interval])  or irate(prometheus_local_storage_persist_errors_total[5m]) ",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Storage Errors ",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        },
                        {
                            "expr": "max_over_time(prometheus_local_storage_rushed_mode[$interval]) or max_over_time(prometheus_local_storage_rushed_mode[5m])",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Has been in Rushed Mode",
                            "refId": "B",
                            "step": 60
                        },
                        {
                            "expr": "prometheus_local_storage_inconsistencies_total",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Storage Inconsitencies Discovered since Start",
                            "refId": "C",
                            "step": 60
                        },
                        {
                            "expr": "rate(prometheus_target_skipped_scrapes_total[$interval]) or rate(prometheus_target_skipped_scrapes_total[5m])",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Scrapes Skipped (Possible Storge Overload)",
                            "refId": "D",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Prometheus Problems",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                }
            ],
            "repeat": null,
            "repeatIteration": null,
            "repeatRowId": null,
            "showTitle": false,
            "title": "New row",
            "titleSize": "h6"
        },
        {
            "collapse": false,
            "height": 250,
            "panels": [
                {
                    "aliasColors": {
                        "CPU Cores Total": "#BF1B00"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 3,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "show": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 2,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 5,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [
                        {
                            "alias": "CPU",
                            "color": "#E24D42"
                        },
                        {
                            "alias": "CPU Cores Used",
                            "fill": 1
                        }
                    ],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "rate(process_cpu_seconds_total{job=\"prometheus\"}[$interval]) or irate(process_cpu_seconds_total{job=\"prometheus\"}[5m])",
                            "format": "time_series",
                            "hide": false,
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "CPU Cores Used",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        },
                        {
                            "expr": "count(node_cpu{mode=\"user\", instance=\"pmm-server\"})",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "CPU Cores Total",
                            "refId": "B",
                            "step": 40
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Prometheus Process CPU Usage",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "cumulative"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "none",
                            "label": "",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "short",
                            "label": "",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Resident Memory": "#629E51",
                        "Total System Memory": "#890F02"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 5,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 2,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 5,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [
                        {
                            "alias": "Memory",
                            "color": "#EF843C"
                        },
                        {
                            "alias": "Resident Memory",
                            "fill": 1
                        }
                    ],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "process_resident_memory_bytes{job=\"prometheus\"}",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Resident Memory",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        },
                        {
                            "expr": "process_virtual_memory_bytes{job=\"prometheus\"}",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Virtual Memory",
                            "refId": "B",
                            "step": 60
                        },
                        {
                            "expr": "go_memstats_alloc_bytes{job=\"prometheus\"}+go_memstats_stack_inuse_bytes{job=\"prometheus\"}",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Go Memory Allocated and In Use",
                            "refId": "C",
                            "step": 60
                        },
                        {
                            "expr": "prometheus_local_storage_target_heap_size_bytes{job=\"prometheus\"}",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 2,
                            "legendFormat": "Configured Target Storage Heap Size",
                            "refId": "D",
                            "step": 120
                        },
                        {
                            "expr": "node_memory_MemTotal{instance=\"pmm-server\"}",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Total System Memory",
                            "refId": "E",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Prometheus Process  Memory Usage",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "cumulative"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "bytes",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839",
                        "Rushed Mode High Watermark": "#BF1B00",
                        "Rushed Mode Low Watermark": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 22,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "total": false,
                        "values": true
                    },
                    "lines": false,
                    "linewidth": 2,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 1,
                    "points": true,
                    "renderer": "flot",
                    "seriesOverrides": [
                        {
                            "alias": "Percent of Time Checkpoint is Active",
                            "yaxis": 2
                        }
                    ],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "rate(prometheus_local_storage_out_of_order_samples_total{instance=\"prometheus\"}[$interval]) or irate(prometheus_local_storage_out_of_order_samples_total{instance=\"prometheus\"}[5m])",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Samples Out of Order - {{reason}}",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Prometheus Out of Order Samples",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "percentunit",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839",
                        "Rushed Mode High Watermark": "#BF1B00",
                        "Rushed Mode Low Watermark": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 18,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 2,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 2,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "max_over_time(prometheus_local_storage_persistence_urgency_score[$interval]) or max_over_time(prometheus_local_storage_persistence_urgency_score[5m])",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Urgency Score ",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        },
                        {
                            "expr": "0.8",
                            "format": "time_series",
                            "intervalFactor": 1,
                            "legendFormat": "Rushed Mode High Watermark",
                            "refId": "B",
                            "step": 20
                        },
                        {
                            "expr": "0.7",
                            "format": "time_series",
                            "intervalFactor": 1,
                            "legendFormat": "Rushed Mode Low Watermark",
                            "refId": "C",
                            "step": 20
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Local Storage Urgency Score",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                }
            ],
            "repeat": null,
            "repeatIteration": null,
            "repeatRowId": null,
            "showTitle": false,
            "title": "Dashboard Row",
            "titleSize": "h6"
        },
        {
            "collapse": false,
            "height": "250px",
            "panels": [
                {
                    "aliasColors": {
                        "Chunks Held in Memory": "#7EB26D",
                        "Chunks not yet Persisted": "#890F02",
                        "Count": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 16,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 2,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 2,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [
                        {
                            "alias": "ChunkDescs Held in Memory",
                            "yaxis": 2
                        }
                    ],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "prometheus_local_storage_chunks_to_persist",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Chunks not yet Persisted",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        },
                        {
                            "expr": "prometheus_local_storage_memory_chunks",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Chunks Held in Memory",
                            "refId": "B",
                            "step": 60
                        },
                        {
                            "expr": "prometheus_local_storage_memory_chunkdescs",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "ChunkDescs Held in Memory",
                            "refId": "C",
                            "step": 60
                        },
                        {
                            "expr": "prometheus_local_storage_memory_chunks-(prometheus_local_storage_chunks_to_persist+prometheus_local_storage_open_head_chunks)",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Evictable Chunks",
                            "refId": "D",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Chunks Memory Details",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 21,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 2,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 2,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "rate(prometheus_local_storage_series_ops_total{job=\"prometheus\"}[$interval]) or irate(prometheus_local_storage_series_ops_total{job=\"prometheus\"}[5m])",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Series Ops - {{type}}",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Series Operations",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "ops",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839",
                        "Rushed Mode High Watermark": "#BF1B00",
                        "Rushed Mode Low Watermark": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 19,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 2,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 2,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "rate(prometheus_local_storage_chunk_ops_total{instance=\"prometheus\"}[$interval]) or irate(prometheus_local_storage_chunk_ops_total{instance=\"prometheus\"}[5m])",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "{{type}} - Chunk Operations ",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        },
                        {
                            "expr": "rate(prometheus_local_storage_chunkdesc_ops_total{instance=\"prometheus\"}[$interval]) or irate(prometheus_local_storage_chunkdesc_ops_total{instance=\"prometheus\"}[5m])",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "{{type}} - ChunkDesc Operations ",
                            "refId": "B",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Prometheus Storage Operations",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "ops",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839",
                        "Rushed Mode High Watermark": "#BF1B00",
                        "Rushed Mode Low Watermark": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 20,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "total": false,
                        "values": true
                    },
                    "lines": false,
                    "linewidth": 2,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 1,
                    "points": true,
                    "renderer": "flot",
                    "seriesOverrides": [
                        {
                            "alias": "Percent of Time Checkpoint is Active",
                            "yaxis": 2
                        }
                    ],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "rate(prometheus_local_storage_checkpointing{instance=\"prometheus\"}[$interval]) or irate(prometheus_local_storage_checkpointing{instance=\"prometheus\"}[5m])",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Percent of Time Checkpoint is Active",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        },
                        {
                            "expr": "rate(prometheus_local_storage_checkpoint_duration_seconds_count{instance=\"prometheus\"}[$interval]) or irate(prometheus_local_storage_checkpoint_duration_seconds_count{instance=\"prometheus\"}[5m])",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Number of Checkpoints Performed",
                            "refId": "B",
                            "step": 60
                        },
                        {
                            "expr": "rate(prometheus_local_storage_checkpoint_series_chunks_written_sum{instance=\"prometheus\"}[$interval]) or irate(prometheus_local_storage_checkpoint_series_chunks_written_sum{instance=\"prometheus\"}[5m])",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Chunks Written by Checkpointing",
                            "refId": "C",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Prometheus Checkpointing",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "percentunit",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839",
                        "Rushed Mode High Watermark": "#BF1B00",
                        "Rushed Mode Low Watermark": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 24,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "sort": "avg",
                        "sortDesc": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 1,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 1,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [
                        {
                            "alias": "Percent of Time Checkpoint is Active",
                            "yaxis": 2
                        }
                    ],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "sum(avg_over_time(up[$interval])) by (job) or sum(avg_over_time(up[5m])) by (job)",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "{{job}}",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Scraped Target by Job",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "percentunit",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839",
                        "Rushed Mode High Watermark": "#BF1B00",
                        "Rushed Mode Low Watermark": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 25,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "sort": "avg",
                        "sortDesc": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 1,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 1,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [
                        {
                            "alias": "Percent of Time Checkpoint is Active",
                            "yaxis": 2
                        }
                    ],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "sum(avg_over_time(up[$interval])) by (instance) or sum(avg_over_time(up[5m])) by (instance)",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "{{instance}}",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Scraped Target by Instance",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "percentunit",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839",
                        "Rushed Mode High Watermark": "#BF1B00",
                        "Rushed Mode Low Watermark": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 26,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "sort": "avg",
                        "sortDesc": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 1,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 1,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [
                        {
                            "alias": "Percent of Time Checkpoint is Active",
                            "yaxis": 2
                        }
                    ],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "rate(prometheus_target_interval_length_seconds_count[$interval]) or irate(prometheus_target_interval_length_seconds_count[5m])",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Rate of {{interval}} scrapes",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Scrapes by Target Frequency",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "ops",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "percentunit",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839",
                        "Rushed Mode High Watermark": "#BF1B00",
                        "Rushed Mode Low Watermark": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 27,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "sort": "avg",
                        "sortDesc": true,
                        "total": false,
                        "values": true
                    },
                    "lines": false,
                    "linewidth": 1,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 1,
                    "points": true,
                    "renderer": "flot",
                    "seriesOverrides": [
                        {
                            "alias": "Percent of Time Checkpoint is Active",
                            "yaxis": 2
                        }
                    ],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "(rate(prometheus_target_interval_length_seconds_sum[$interval]) / rate(prometheus_target_interval_length_seconds_count[$interval])) or (irate(prometheus_target_interval_length_seconds_sum[5m]) / irate(prometheus_target_interval_length_seconds_count[5m]))",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Actual  {{interval}} scrape length",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Scrapes by Target Frequency",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "s",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "percentunit",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839",
                        "Max Queries Active": "#E24D42",
                        "Maximum Queries Permitted": "#890F02",
                        "Rushed Mode High Watermark": "#BF1B00",
                        "Rushed Mode Low Watermark": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 28,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "sort": "avg",
                        "sortDesc": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 1,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 1,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [
                        {
                            "alias": "Max Queries Active",
                            "lines": false,
                            "pointradius": 1,
                            "points": true
                        }
                    ],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "avg_over_time(prometheus_engine_queries[$interval]) or avg_over_time(prometheus_engine_queries[5m])",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Avg Queries Active",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        },
                        {
                            "expr": "max_over_time(prometheus_engine_queries[$interval]) or max_over_time(prometheus_engine_queries[5m])",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "Max Queries Active",
                            "refId": "B",
                            "step": 60
                        },
                        {
                            "expr": "prometheus_engine_queries_concurrent_max",
                            "format": "time_series",
                            "intervalFactor": 2,
                            "legendFormat": "Maximum Queries Permitted",
                            "refId": "C",
                            "step": 40
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Prometheus Queries",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "percentunit",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839",
                        "Max Queries Active": "#E24D42",
                        "Maximum Queries Permitted": "#890F02",
                        "Rushed Mode High Watermark": "#BF1B00",
                        "Rushed Mode Low Watermark": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 29,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "sort": "avg",
                        "sortDesc": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 1,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 1,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [
                        {
                            "alias": "Max Queries Active",
                            "lines": false,
                            "pointradius": 1,
                            "points": true
                        }
                    ],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "rate(prometheus_engine_query_duration_seconds_count[$interval]) or irate(prometheus_engine_query_duration_seconds_count[$interval]) ",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "{{slice}} - Executed",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Prometheus Query Execution",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "ops",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "percentunit",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839",
                        "Max Queries Active": "#E24D42",
                        "Maximum Queries Permitted": "#890F02",
                        "Rushed Mode High Watermark": "#BF1B00",
                        "Rushed Mode Low Watermark": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 30,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "sort": "avg",
                        "sortDesc": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 1,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 1,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [
                        {
                            "alias": "Max Queries Active",
                            "lines": false,
                            "pointradius": 1,
                            "points": true
                        }
                    ],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": true,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "(rate(prometheus_engine_query_duration_seconds_sum[$interval])/rate(prometheus_engine_query_duration_seconds_count[$interval])) or (irate(prometheus_engine_query_duration_seconds_sum[$interval])/irate(prometheus_engine_query_duration_seconds_count[$interval]))",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "{{slice}} - Time",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Prometheus Query Execution Performance",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "s",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "percentunit",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                },
                {
                    "aliasColors": {
                        "Count": "#EAB839",
                        "Max Queries Active": "#E24D42",
                        "Maximum Queries Permitted": "#890F02",
                        "Rushed Mode High Watermark": "#BF1B00",
                        "Rushed Mode Low Watermark": "#EAB839"
                    },
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 0,
                    "grid": {},
                    "id": 31,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "max": true,
                        "min": true,
                        "rightSide": false,
                        "show": true,
                        "sort": "avg",
                        "sortDesc": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 1,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 1,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [
                        {
                            "alias": "Max Queries Active",
                            "lines": false,
                            "pointradius": 1,
                            "points": true
                        }
                    ],
                    "spaceLength": 10,
                    "span": 3,
                    "stack": true,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "rate(prometheus_engine_query_duration_seconds_sum[$interval])",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "{{slice}} - Load",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "Prometheus Query Execution Performance",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 0,
                        "value_type": "individual"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "short",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "percentunit",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                }
            ],
            "repeat": null,
            "repeatIteration": null,
            "repeatRowId": null,
            "showTitle": false,
            "title": "Prometheus Statistics",
            "titleSize": "h6"
        },
        {
            "collapse": false,
            "height": "250px",
            "panels": [
                {
                    "aliasColors": {},
                    "bars": false,
                    "dashLength": 10,
                    "dashes": false,
                    "datasource": "Prometheus",
                    "decimals": 2,
                    "editable": true,
                    "error": false,
                    "fill": 2,
                    "grid": {},
                    "id": 1,
                    "legend": {
                        "alignAsTable": true,
                        "avg": true,
                        "current": false,
                        "hideEmpty": false,
                        "hideZero": true,
                        "max": true,
                        "min": true,
                        "rightSide": true,
                        "show": true,
                        "sort": "avg",
                        "sortDesc": true,
                        "total": false,
                        "values": true
                    },
                    "lines": true,
                    "linewidth": 2,
                    "links": [],
                    "nullPointMode": "null",
                    "percentage": false,
                    "pointradius": 5,
                    "points": false,
                    "renderer": "flot",
                    "seriesOverrides": [],
                    "spaceLength": 10,
                    "span": 12,
                    "stack": false,
                    "steppedLine": false,
                    "targets": [
                        {
                            "expr": "sum(rate(http_requests_total{job=\"prometheus\"}[$interval])) by (handler) or sum(irate(http_requests_total{job=\"prometheus\"}[5m])) by (handler)",
                            "format": "time_series",
                            "interval": "$interval",
                            "intervalFactor": 1,
                            "legendFormat": "{{ handler }}",
                            "metric": "",
                            "refId": "A",
                            "step": 60
                        }
                    ],
                    "thresholds": [],
                    "timeFrom": null,
                    "timeShift": null,
                    "title": "HTTP Requests by Handler",
                    "tooltip": {
                        "msResolution": false,
                        "shared": true,
                        "sort": 2,
                        "value_type": "cumulative"
                    },
                    "type": "graph",
                    "xaxis": {
                        "buckets": null,
                        "mode": "time",
                        "name": null,
                        "show": true,
                        "values": []
                    },
                    "yaxes": [
                        {
                            "format": "short",
                            "label": "",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        },
                        {
                            "format": "short",
                            "label": "",
                            "logBase": 1,
                            "max": null,
                            "min": 0,
                            "show": true
                        }
                    ]
                }
            ],
            "repeat": null,
            "repeatIteration": null,
            "repeatRowId": null,
            "showTitle": false,
            "title": "New row",
            "titleSize": "h6"
        },
        {
            "collapse": false,
            "height": "250px",
            "panels": [
                {
                    "columns": [
                        {
                            "text": "Current",
                            "value": "current"
                        }
                    ],
                    "datasource": "Prometheus",
                    "editable": true,
                    "error": false,
                    "fontSize": "100%",
                    "height": "420px",
                    "hideTimeOverride": true,
                    "id": 13,
                    "links": [],
                    "pageSize": null,
                    "scroll": false,
                    "showHeader": true,
                    "sort": {
                        "col": 1,
                        "desc": true
                    },
                    "span": 6,
                    "styles": [],
                    "targets": [
                        {
                            "expr": "topk(10, count({__name__=~\".+\"})  by (__name__))",
                            "interval": "5m",
                            "intervalFactor": 2,
                            "legendFormat": "{{ __name__ }}",
                            "refId": "A",
                            "step": 600
                        }
                    ],
                    "timeFrom": "5m",
                    "title": "Top 10 metrics by time series count",
                    "transform": "timeseries_aggregations",
                    "type": "table"
                },
                {
                    "columns": [
                        {
                            "text": "Current",
                            "value": "current"
                        }
                    ],
                    "datasource": "Prometheus",
                    "editable": true,
                    "error": false,
                    "fontSize": "100%",
                    "height": "420px",
                    "hideTimeOverride": true,
                    "id": 14,
                    "links": [],
                    "pageSize": null,
                    "scroll": false,
                    "showHeader": true,
                    "sort": {
                        "col": 1,
                        "desc": true
                    },
                    "span": 6,
                    "styles": [],
                    "targets": [
                        {
                            "expr": "topk(10, count({__name__=~\".+\"})  by (instance))",
                            "interval": "5m",
                            "intervalFactor": 2,
                            "legendFormat": "{{ instance }}",
                            "refId": "A",
                            "step": 600
                        }
                    ],
                    "timeFrom": "5m",
                    "title": "Top 10 hosts by time series count",
                    "transform": "timeseries_aggregations",
                    "type": "table"
                }
            ],
            "repeat": null,
            "repeatIteration": null,
            "repeatRowId": null,
            "showTitle": false,
            "title": "New row",
            "titleSize": "h6"
        }
    ],
    "schemaVersion": 14,
    "style": "dark",
    "tags": [
        "Percona"
    ],
    "templating": {
        "list": [
            {
                "allFormat": "glob",
                "auto": true,
                "auto_count": 200,
                "auto_min": "1s",
                "current": {
                    "text": "auto",
                    "value": "$__auto_interval"
                },
                "datasource": "Prometheus",
                "hide": 0,
                "includeAll": false,
                "label": "Interval",
                "multi": false,
                "multiFormat": "glob",
                "name": "interval",
                "options": [
                    {
                        "selected": true,
                        "text": "auto",
                        "value": "$__auto_interval"
                    },
                    {
                        "selected": false,
                        "text": "1s",
                        "value": "1s"
                    },
                    {
                        "selected": false,
                        "text": "5s",
                        "value": "5s"
                    },
                    {
                        "selected": false,
                        "text": "1m",
                        "value": "1m"
                    },
                    {
                        "selected": false,
                        "text": "5m",
                        "value": "5m"
                    },
                    {
                        "selected": false,
                        "text": "1h",
                        "value": "1h"
                    },
                    {
                        "selected": false,
                        "text": "6h",
                        "value": "6h"
                    },
                    {
                        "selected": false,
                        "text": "1d",
                        "value": "1d"
                    }
                ],
                "query": "1s,5s,1m,5m,1h,6h,1d",
                "refresh": 2,
                "type": "interval"
            }
        ]
    },
    "time": {
        "from": "now-12h",
        "to": "now"
    },
    "timepicker": {
        "now": true,
        "refresh_intervals": [
            "5s",
            "10s",
            "30s",
            "1m",
            "5m",
            "15m",
            "30m",
            "1h",
            "2h",
            "1d"
        ],
        "time_options": [
            "5m",
            "15m",
            "1h",
            "6h",
            "12h",
            "24h",
            "2d",
            "7d",
            "30d"
        ]
    },
    "timezone": "browser",
    "title": "Prometheus",
    "version": 0
}