785 lines
41 KiB
JSON
785 lines
41 KiB
JSON
{
|
|
"annotations":{"list":[{"builtIn":1,"datasource":{"type":"datasource","uid":"grafana"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations & Alerts","target":{"limit":100,"matchAny":false,"tags":[],"type":"dashboard"},"type":"dashboard"}]},
|
|
"author":"Ruohang Feng (rh@vonng.com)",
|
|
"description":"Node key metrics and alerts",
|
|
"editable":true,
|
|
"fiscalYearStartMonth":0,
|
|
"graphTooltip":0,
|
|
"id":null,
|
|
"license":"https://pigsty.io/docs/about/license/",
|
|
"links":[{"asDropdown":true,"icon":"external link","includeVars":true,"keepTime":true,"tags":["Pigsty","NODE"],"targetBlank":false,"title":"NODE","tooltip":"","type":"dashboards","url":""}],
|
|
"panels":[
|
|
{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":16,"panels":[],"title":"Overview","type":"row"},
|
|
{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"description":"",
|
|
"fieldConfig":{
|
|
"defaults":{"color":{"mode":"thresholds"},"decimals":0,"links":[],"mappings":[],"max":1,"min":0,"noValue":"OK","thresholds":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#fcdb72","value":1},{"color":"#f79f64","value":4},{"color":"#b783af","value":8},{"color":"text","value":16}]},"unit":"none"},
|
|
"overrides":[{"matcher":{"id":"byFrameRefID","options":"A"},"properties":[{"id":"links","value":[{"targetBlank":true,"title":"Pigsty Alertmanager","url":"/alertmgr/"}]}]},{"matcher":{"id":"byFrameRefID","options":"B"},"properties":[{"id":"links","value":[{"title":"PGSQL Alert","url":"/d/pgsql-alert?${__url_time_range}"}]}]}]
|
|
},
|
|
"gridPos":{"h":5,"w":2,"x":0,"y":1},
|
|
"id":87,
|
|
"options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{},"textMode":"auto","wideLayout":true},
|
|
"pluginVersion":"12.3.0",
|
|
"targets":[
|
|
{ "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(ALERTS{alertstate=\"firing\", job=\"node\"}) or on() vector(0)", "interval":"","legendFormat":"Firing Alert" ,"range":true,"refId":"A" },
|
|
{ "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(ALERTS{alertstate=\"pending\", job=\"node\"})" ,"hide":true,"interval":"","legendFormat":"Pending Alert","range":true,"refId":"B" }
|
|
],
|
|
"type":"stat"
|
|
},
|
|
{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"description":"",
|
|
"fieldConfig":{
|
|
"defaults":{
|
|
"color":{"mode":"thresholds"},
|
|
"decimals":0,
|
|
"links":[{"targetBlank":true,"title":"AlertInfo ${__field.labels.alertname}","url":"/alertmgr/#/alerts?filter=%7Balertname%3D%22${__field.labels.alertname}%22%7D"}],
|
|
"mappings":[],
|
|
"max":1,
|
|
"min":0,
|
|
"noValue":"OK",
|
|
"thresholds":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"#fcdb72","value":1},{"color":"#f79f64","value":2},{"color":"#b783af","value":4}]},
|
|
"unit":"none"
|
|
},
|
|
"overrides":[]
|
|
},
|
|
"gridPos":{"h":5,"w":4,"x":2,"y":1},
|
|
"id":86,
|
|
"options":{"colorMode":"background","graphMode":"area","justifyMode":"auto","orientation":"horizontal","percentChangeColorMode":"standard","reduceOptions":{"calcs":["last"],"fields":"","values":false},"showPercentChange":false,"text":{"titleSize":12},"textMode":"auto","wideLayout":true},
|
|
"pluginVersion":"12.3.0",
|
|
"targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"count(ALERTS{alertstate=\"firing\", job=\"node\"}) by (alertname, severity)","interval":"","legendFormat":"{{ alertname }}","range":true,"refId":"A"}],
|
|
"type":"stat"
|
|
},
|
|
{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"description":"",
|
|
"fieldConfig":{
|
|
"defaults":{
|
|
"color":{"mode":"palette-classic"},
|
|
"custom":{
|
|
"axisBorderShow":false,
|
|
"axisCenteredZero":false,
|
|
"axisColorMode":"text",
|
|
"axisLabel":"",
|
|
"axisPlacement":"hidden",
|
|
"axisSoftMax":1.2,
|
|
"axisSoftMin":0,
|
|
"barAlignment":0,
|
|
"barWidthFactor":0.6,
|
|
"drawStyle":"line",
|
|
"fillOpacity":98,
|
|
"gradientMode":"none",
|
|
"hideFrom":{"graph":false,"legend":false,"tooltip":false,"viz":false},
|
|
"insertNulls":false,
|
|
"lineInterpolation":"stepBefore",
|
|
"lineWidth":0,
|
|
"pointSize":1,
|
|
"scaleDistribution":{"type":"linear"},
|
|
"showPoints":"never",
|
|
"showValues":false,
|
|
"spanNulls":false,
|
|
"stacking":{"group":"A","mode":"normal"},
|
|
"thresholdsStyle":{"mode":"off"}
|
|
},
|
|
"decimals":0,
|
|
"links":[
|
|
{ "targetBlank":true,"title":"AlertInfo ${__field.labels.alertname} on Cluster ${__field.labels.cls}","url":"/alertmgr/#/alerts?filter=%7Bcls%3D%22${__field.labels.cls}%22%2C%20alertname%3D%22${__field.labels.alertname}%22%7D" },
|
|
{ "targetBlank":true,"title":"Silence ${__field.labels.alertname} on Cluster ${__field.labels.cls}" ,"url":"/alertmgr/#/silences/new?filter=%7Bcls%3D%22${__field.labels.cls}%22%2C%20alertname%3D%22${__field.labels.alertname}%22%7D" },
|
|
{ "title":"Node Cluster for ${__field.labels.cls} : Alerts" ,"url":"/d/node-cluster?var-cls=${__field.labels.cls}&viewPanel=227&${__url_time_range}" }
|
|
],
|
|
"mappings":[],
|
|
"min":0,
|
|
"thresholds":{"mode":"absolute","steps":[{"color":"#346f36cc","value":0}]},
|
|
"unit":"none"
|
|
},
|
|
"overrides":[{"matcher":{"id":"byFrameRefID","options":"A"},"properties":[{"id":"custom.fillOpacity","value":66}]},{"matcher":{"id":"byType","options":"time"},"properties":[{"id":"custom.axisPlacement","value":"auto"}]}]
|
|
},
|
|
"gridPos":{"h":5,"w":18,"x":6,"y":1},
|
|
"id":34,
|
|
"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
|
|
"pluginVersion":"12.3.0",
|
|
"targets":[
|
|
{ "datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"sum without (ins,ip,instance,datname) (ALERTS{alertstate=\"firing\", job=\"node\"})", "interval":"","legendFormat":"[{{ severity }}] {{alertname}} @ {{ cls }}","range":true,"refId":"A" },
|
|
{ "datasource":{"type":"prometheus","uid":"ds-prometheus"}, "exemplar":false,"expr":"0" ,"hide":true,"interval":"","legendFormat":"Alert" , "refId":"C" }
|
|
],
|
|
"title":"Firing Alerts",
|
|
"type":"timeseries"
|
|
},
|
|
{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"description":"",
|
|
"fieldConfig":{
|
|
"defaults":{"color":{"mode":"thresholds"},"custom":{"align":"center","cellOptions":{"type":"auto"},"footer":{"reducers":[]},"inspect":false},"decimals":0,"links":[],"mappings":[],"max":-4,"min":0,"thresholds":{"mode":"absolute","steps":[{"color":"#e3e3e3e0","value":0}]},"unit":"none"},
|
|
"overrides":[
|
|
{"matcher":{"id":"byName","options":"Alert"},"properties":[{"id":"custom.width"},{"id":"links","value":[{"targetBlank":true,"title":"AlertInfo for ${__field.labels.alertname} on Cluster ${__field.labels.cls}","url":"/alertmgr/#/alerts?filter=%7Bcls%3D%22${__data.fields.Cluster}%22%2C%20alertname%3D%22${__data.fields.Alert}%22%7D"}]}]},
|
|
{"matcher":{"id":"byName","options":"State"},"properties":[{"id":"custom.width","value":80},{"id":"mappings","value":[{"options":{"firing":{"color":"light-red","index":0,"text":"Firing\ud83d\udd25"},"pending":{"color":"light-orange","index":1,"text":"Pend\u23f0"}},"type":"value"}]},{"id":"custom.cellOptions","value":{"mode":"gradient","type":"color-background"}}]},
|
|
{"matcher":{"id":"byName","options":"Level"},"properties":[{"id":"custom.width","value":60},{"id":"mappings","value":[{"options":{"0":{"color":"red","index":0,"text":"P0"},"1":{"color":"#f79f64","index":1,"text":"P1"},"2":{"color":"#fcdb72","index":2,"text":"P2"}},"type":"value"}]},{"id":"custom.cellOptions","value":{"mode":"gradient","type":"color-background"}}]},
|
|
{
|
|
"matcher":{"id":"byName","options":"Severity"},
|
|
"properties":[{"id":"custom.width","value":110},{"id":"mappings","value":[{"options":{"CRIT":{"color":"red","index":0,"text":"Critical \u26a1\ufe0f"},"INFO":{"color":"light-yellow","index":2,"text":"Info \u2139\ufe0f"},"WARN":{"color":"super-light-orange","index":1,"text":"Warning \u26a0\ufe0f"}},"type":"value"}]},{"id":"custom.cellOptions","value":{"mode":"gradient","type":"color-background"}}]
|
|
},
|
|
{"matcher":{"id":"byName","options":"Category"},"properties":[{"id":"custom.width","value":80},{"id":"mappings","value":[{"options":{"infra":{"color":"#f79f64","index":2,"text":"INFRA"},"node":{"color":"#346f36cc","index":1,"text":"NODE"},"pgsql":{"color":"light-blue","index":0,"text":"PGSQL"}},"type":"value"}]},{"id":"custom.cellOptions","value":{"mode":"gradient","type":"color-background"}}]},
|
|
{"matcher":{"id":"byName","options":"Ins"},"properties":[{"id":"custom.width","value":100}]},{"matcher":{"id":"byName","options":"Active"},"properties":[{"id":"custom.width","value":66}]},
|
|
{"matcher":{"id":"byName","options":"Cluster"},"properties":[{"id":"custom.width"},{"id":"links","value":[{"title":"PGSQL Cluster for ${cls}","url":"/d/pgsql-cluster?var-cls=${__data.fields.Cluster}&${__url_time_range}"}]}]},
|
|
{"matcher":{"id":"byName","options":"Instance"},"properties":[{"id":"links","value":[{"title":"PGSQL Instance for ${__data.fields.Instance}","url":"/d/pgsql-instance?var-ins=${__data.fields.Instance}&${__url_time_range}"}]}]},
|
|
{"matcher":{"id":"byName","options":"Database"},"properties":[{"id":"custom.width","value":80},{"id":"links","value":[{"title":"PGSQL Database for ${__data.fields.Database} on ${__data.fields.Instance}","url":"/d/pgsql-database?var-ins=${__data.fields.Instance}&var-datname=${__data.fields.Database}&${__url_time_range}"}]}]},
|
|
{
|
|
"matcher":{"id":"byName","options":"Silence"},
|
|
"properties":[{"id":"links","value":[{"targetBlank":true,"title":"Silence ${__data.fields.Alert} on Cluster ${__data.fields.Cluster}","url":"/alertmgr/#/silences/new?filter=%7Bcls%3D%22${__data.fields.Cluster}%22%2C%20alertname%3D%22${__data.fields.Alert}%22%7D"}]},{"id":"mappings","value":[{"options":{"1":{"index":0,"text":"silence"}},"type":"value"}]},{"id":"custom.width","value":70}]
|
|
},
|
|
{"matcher":{"id":"byName","options":"Agent"},"properties":[{"id":"custom.width","value":140}]},{"matcher":{"id":"byName","options":"IP"},"properties":[{"id":"custom.width","value":110},{"id":"links","value":[{"title":"PGSQL Node: ${__data.fields.IP}","url":"/d/node-instance?var-id=${__data.fields.IP}&${__url_time_range}"}]}]}
|
|
]
|
|
},
|
|
"gridPos":{"h":8,"w":24,"x":0,"y":6},
|
|
"id":36,
|
|
"options":{"cellHeight":"sm","showHeader":true,"sortBy":[{"desc":false,"displayName":"Level"}]},
|
|
"pluginVersion":"12.3.0",
|
|
"targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"ALERTS{job=\"node\"}","format":"table","instant":true,"interval":"","legendFormat":"","refId":"A"}],
|
|
"title":"Active Alerts",
|
|
"transformations":[
|
|
{
|
|
"id":"organize",
|
|
"options":{
|
|
"excludeByName":{"Time":true,"Value":false,"__name__":true,"job":true},
|
|
"indexByName":{"Time":11,"Value":10,"__name__":13,"alertname":0,"alertstate":1,"category":4,"cls":5,"datname":8,"ins":6,"instance":9,"ip":7,"job":12,"level":2,"severity":3},
|
|
"renameByName":{"Value":"Silence","Value #A":"Silence","alertname":"Alert","alertstate":"State","category":"Category","cls":"Cluster","datname":"Database","ins":"Instance","instance":"Agent","ip":"IP","job":"","level":"Level","severity":"Severity"}
|
|
}
|
|
}
|
|
],
|
|
"type":"table"
|
|
},
|
|
{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"fieldConfig":{
|
|
"defaults":{
|
|
"color":{"mode":"palette-classic"},
|
|
"custom":{
|
|
"axisBorderShow":false,
|
|
"axisCenteredZero":false,
|
|
"axisColorMode":"text",
|
|
"axisLabel":"",
|
|
"axisPlacement":"auto",
|
|
"axisSoftMin":0,
|
|
"barAlignment":0,
|
|
"barWidthFactor":0.6,
|
|
"drawStyle":"line",
|
|
"fillOpacity":0,
|
|
"gradientMode":"none",
|
|
"hideFrom":{"legend":false,"tooltip":false,"viz":false},
|
|
"insertNulls":false,
|
|
"lineInterpolation":"linear",
|
|
"lineWidth":1,
|
|
"pointSize":5,
|
|
"scaleDistribution":{"type":"linear"},
|
|
"showPoints":"auto",
|
|
"showValues":false,
|
|
"spanNulls":false,
|
|
"stacking":{"group":"A","mode":"none"},
|
|
"thresholdsStyle":{"mode":"area"}
|
|
},
|
|
"decimals":1,
|
|
"links":[{"title":"PGSQL Node : CPU for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=172&${__url_time_range}"}],
|
|
"mappings":[],
|
|
"max":1,
|
|
"min":0,
|
|
"thresholds":{"mode":"absolute","steps":[{"color":"#3e668f","value":0},{"color":"green","value":0.1},{"color":"yellow","value":0.3},{"color":"orange","value":0.5},{"color":"red","value":0.7},{"color":"purple","value":0.9},{"color":"text","value":1}]},
|
|
"unit":"percentunit"
|
|
},
|
|
"overrides":[]
|
|
},
|
|
"gridPos":{"h":8,"w":24,"x":0,"y":14},
|
|
"id":18,
|
|
"options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"hideZeros":false,"mode":"multi","sort":"none"}},
|
|
"pluginVersion":"12.3.0",
|
|
"targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"node:ins:cpu_usage_1m","interval":"","legendFormat":"{{ ip }}","refId":"A"}],
|
|
"title":"NodeCpuHigh",
|
|
"type":"timeseries"
|
|
},
|
|
{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":22},"id":82,"panels":[],"title":"Error Logs","type":"row"},
|
|
{
|
|
"datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},
|
|
"description":"Filter syslog with priority < 4, emerg, alert, crit, error",
|
|
"gridPos":{"h":13,"w":24,"x":0,"y":23},
|
|
"id":84,
|
|
"maxDataPoints":100,
|
|
"options":{"dedupStrategy":"none","detailsMode":"sidebar","enableInfiniteScrolling":true,"enableLogDetails":true,"fontSize":"small","prettifyLogMessage":false,"showCommonLabels":false,"showControls":true,"showLabels":true,"showTime":true,"sortOrder":"Descending","wrapLogMessage":false},
|
|
"pluginVersion":"12.3.0",
|
|
"targets":[{"datasource":{"type":"victoriametrics-logs-datasource","uid":"ds-vlogs"},"editorMode":"code","expr":"job:syslog p:<4 | fields level,unit,ins,ip,_msg","queryType":"instant","refId":"A"}],
|
|
"title":"Node Syslog Error",
|
|
"type":"logs"
|
|
},
|
|
{
|
|
"collapsed":true,
|
|
"gridPos":{"h":1,"w":24,"x":0,"y":36},
|
|
"id":53,
|
|
"panels":[
|
|
{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"description":"Standard Load (divide by cpu count) > 100%",
|
|
"fieldConfig":{
|
|
"defaults":{
|
|
"color":{"mode":"palette-classic"},
|
|
"custom":{
|
|
"axisCenteredZero":false,
|
|
"axisColorMode":"text",
|
|
"axisLabel":"",
|
|
"axisPlacement":"auto",
|
|
"axisSoftMax":1,
|
|
"axisSoftMin":0,
|
|
"barAlignment":0,
|
|
"drawStyle":"line",
|
|
"fillOpacity":0,
|
|
"gradientMode":"none",
|
|
"hideFrom":{"legend":false,"tooltip":false,"viz":false},
|
|
"lineInterpolation":"linear",
|
|
"lineWidth":1,
|
|
"pointSize":5,
|
|
"scaleDistribution":{"type":"linear"},
|
|
"showPoints":"auto",
|
|
"spanNulls":false,
|
|
"stacking":{"group":"A","mode":"none"},
|
|
"thresholdsStyle":{"mode":"area"}
|
|
},
|
|
"decimals":1,
|
|
"links":[{"title":"PGSQL Node: Load for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=173&${__url_time_range}"}],
|
|
"mappings":[],
|
|
"min":0,
|
|
"thresholds":{"mode":"absolute","steps":[{"color":"#3e668f"},{"color":"green","value":0.1},{"color":"yellow","value":0.3},{"color":"orange","value":0.5},{"color":"red","value":0.7},{"color":"purple","value":0.9},{"color":"text","value":1}]},
|
|
"unit":"percentunit"
|
|
},
|
|
"overrides":[]
|
|
},
|
|
"gridPos":{"h":8,"w":24,"x":0,"y":55},
|
|
"id":11,
|
|
"options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
|
|
"targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"node:ins:stdload1","interval":"","legendFormat":"{{ ip }}","refId":"A"}],
|
|
"title":"NodeLoadHigh",
|
|
"type":"timeseries"
|
|
}
|
|
],
|
|
"title":"Node: Schedule",
|
|
"type":"row"
|
|
},
|
|
{
|
|
"collapsed":true,
|
|
"gridPos":{"h":1,"w":24,"x":0,"y":37},
|
|
"id":85,
|
|
"panels":[
|
|
{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"description":"Available memory < 10%",
|
|
"fieldConfig":{
|
|
"defaults":{
|
|
"color":{"mode":"palette-classic"},
|
|
"custom":{
|
|
"axisCenteredZero":false,
|
|
"axisColorMode":"text",
|
|
"axisLabel":"",
|
|
"axisPlacement":"auto",
|
|
"axisSoftMin":0,
|
|
"barAlignment":0,
|
|
"drawStyle":"line",
|
|
"fillOpacity":0,
|
|
"gradientMode":"none",
|
|
"hideFrom":{"legend":false,"tooltip":false,"viz":false},
|
|
"lineInterpolation":"linear",
|
|
"lineWidth":1,
|
|
"pointSize":5,
|
|
"scaleDistribution":{"type":"linear"},
|
|
"showPoints":"auto",
|
|
"spanNulls":false,
|
|
"stacking":{"group":"A","mode":"none"},
|
|
"thresholdsStyle":{"mode":"area"}
|
|
},
|
|
"decimals":1,
|
|
"links":[{"title":"PGSQL Node: Memory for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=40&${__url_time_range}"}],
|
|
"mappings":[],
|
|
"max":1,
|
|
"min":0,
|
|
"thresholds":{"mode":"absolute","steps":[{"color":"purple"},{"color":"red","value":0.1},{"color":"orange","value":0.15},{"color":"green","value":0.2}]},
|
|
"unit":"percentunit"
|
|
},
|
|
"overrides":[]
|
|
},
|
|
"gridPos":{"h":8,"w":24,"x":0,"y":64},
|
|
"id":56,
|
|
"options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
|
|
"targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"node:ins:mem_avail","interval":"","legendFormat":"{{ ip }}","range":true,"refId":"A"}],
|
|
"title":"NodeOutOfMem",
|
|
"type":"timeseries"
|
|
}
|
|
],
|
|
"title":"Node: Memory",
|
|
"type":"row"
|
|
},
|
|
{
|
|
"collapsed":true,
|
|
"gridPos":{"h":1,"w":24,"x":0,"y":38},
|
|
"id":55,
|
|
"panels":[
|
|
{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"description":"Available memory < 10%",
|
|
"fieldConfig":{
|
|
"defaults":{
|
|
"color":{"mode":"palette-classic"},
|
|
"custom":{
|
|
"axisCenteredZero":false,
|
|
"axisColorMode":"text",
|
|
"axisLabel":"",
|
|
"axisPlacement":"auto",
|
|
"axisSoftMin":0,
|
|
"barAlignment":0,
|
|
"drawStyle":"line",
|
|
"fillOpacity":0,
|
|
"gradientMode":"none",
|
|
"hideFrom":{"legend":false,"tooltip":false,"viz":false},
|
|
"lineInterpolation":"linear",
|
|
"lineWidth":1,
|
|
"pointSize":5,
|
|
"scaleDistribution":{"type":"linear"},
|
|
"showPoints":"auto",
|
|
"spanNulls":false,
|
|
"stacking":{"group":"A","mode":"none"},
|
|
"thresholdsStyle":{"mode":"area"}
|
|
},
|
|
"links":[{"title":"PGSQL Node: Swap for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=33&${__url_time_range}"}],
|
|
"mappings":[],
|
|
"min":0,
|
|
"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":0.01},{"color":"red","value":0.05},{"color":"purple","value":0.1}]},
|
|
"unit":"percentunit"
|
|
},
|
|
"overrides":[]
|
|
},
|
|
"gridPos":{"h":8,"w":24,"x":0,"y":73},
|
|
"id":57,
|
|
"options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
|
|
"targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"node:ins:swap_usage","interval":"","legendFormat":"{{ ip }}","refId":"A"}],
|
|
"title":"NodeMemSwapped",
|
|
"type":"timeseries"
|
|
}
|
|
],
|
|
"title":"Node: Swap",
|
|
"type":"row"
|
|
},
|
|
{
|
|
"collapsed":true,
|
|
"gridPos":{"h":1,"w":24,"x":0,"y":39},
|
|
"id":27,
|
|
"panels":[
|
|
{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"description":"Disk read/write avg latency > 32ms",
|
|
"fieldConfig":{
|
|
"defaults":{
|
|
"color":{"mode":"palette-classic"},
|
|
"custom":{
|
|
"axisCenteredZero":false,
|
|
"axisColorMode":"text",
|
|
"axisLabel":"",
|
|
"axisPlacement":"auto",
|
|
"axisSoftMin":0,
|
|
"barAlignment":0,
|
|
"drawStyle":"line",
|
|
"fillOpacity":0,
|
|
"gradientMode":"none",
|
|
"hideFrom":{"legend":false,"tooltip":false,"viz":false},
|
|
"lineInterpolation":"linear",
|
|
"lineWidth":1,
|
|
"pointSize":5,
|
|
"scaleDistribution":{"type":"linear"},
|
|
"showPoints":"auto",
|
|
"spanNulls":false,
|
|
"stacking":{"group":"A","mode":"none"},
|
|
"thresholdsStyle":{"mode":"area"}
|
|
},
|
|
"links":[{"title":"PGSQL Node: Disk Latency for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=29&${__url_time_range}"}],
|
|
"mappings":[],
|
|
"min":0,
|
|
"thresholds":{"mode":"absolute","steps":[{"color":"#3e668f"},{"color":"green","value":0.001},{"color":"orange","value":0.008},{"color":"red","value":0.032},{"color":"purple","value":0.128},{"color":"text","value":1}]},
|
|
"unit":"s"
|
|
},
|
|
"overrides":[]
|
|
},
|
|
"gridPos":{"h":8,"w":24,"x":0,"y":82},
|
|
"id":60,
|
|
"options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
|
|
"targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"max by (ip,device) (node:dev:disk_write_rt_1m > node:dev:disk_read_rt_1m or node:dev:disk_read_rt_1m)","interval":"","legendFormat":"{{ ip}}-{{ device }}","refId":"A"}],
|
|
"title":"NodeDiskSlow",
|
|
"type":"timeseries"
|
|
}
|
|
],
|
|
"title":"NODE: Disk",
|
|
"type":"row"
|
|
},
|
|
{
|
|
"collapsed":true,
|
|
"gridPos":{"h":1,"w":24,"x":0,"y":40},
|
|
"id":29,
|
|
"panels":[
|
|
{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"fieldConfig":{
|
|
"defaults":{
|
|
"color":{"mode":"palette-classic"},
|
|
"custom":{
|
|
"axisCenteredZero":false,
|
|
"axisColorMode":"text",
|
|
"axisLabel":"",
|
|
"axisPlacement":"auto",
|
|
"axisSoftMax":1,
|
|
"axisSoftMin":0,
|
|
"barAlignment":0,
|
|
"drawStyle":"line",
|
|
"fillOpacity":0,
|
|
"gradientMode":"none",
|
|
"hideFrom":{"legend":false,"tooltip":false,"viz":false},
|
|
"lineInterpolation":"linear",
|
|
"lineWidth":1,
|
|
"pointSize":5,
|
|
"scaleDistribution":{"type":"linear"},
|
|
"showPoints":"auto",
|
|
"spanNulls":false,
|
|
"stacking":{"group":"A","mode":"none"},
|
|
"thresholdsStyle":{"mode":"area"}
|
|
},
|
|
"links":[{"title":"PGSQL Node: Space Usage for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=199&${__url_time_range}"}],
|
|
"mappings":[],
|
|
"max":3,
|
|
"min":0,
|
|
"thresholds":{"mode":"absolute","steps":[{"color":"#3e668f"},{"color":"green","value":0.1},{"color":"yellow","value":0.8},{"color":"orange","value":0.9},{"color":"red","value":0.95},{"color":"purple","value":0.99},{"color":"text","value":1}]},
|
|
"unit":"percentunit"
|
|
},
|
|
"overrides":[]
|
|
},
|
|
"gridPos":{"h":8,"w":24,"x":0,"y":91},
|
|
"id":23,
|
|
"options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
|
|
"targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"node:fs:space_usage","interval":"","legendFormat":"{{ ip }}:{{ mountpoint }}","refId":"A"}],
|
|
"title":"NodeFsFull",
|
|
"type":"timeseries"
|
|
},{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"fieldConfig":{
|
|
"defaults":{
|
|
"color":{"mode":"palette-classic"},
|
|
"custom":{
|
|
"axisCenteredZero":false,
|
|
"axisColorMode":"text",
|
|
"axisLabel":"",
|
|
"axisPlacement":"auto",
|
|
"barAlignment":0,
|
|
"drawStyle":"line",
|
|
"fillOpacity":0,
|
|
"gradientMode":"opacity",
|
|
"hideFrom":{"legend":false,"tooltip":false,"viz":false},
|
|
"lineInterpolation":"linear",
|
|
"lineWidth":1,
|
|
"pointSize":5,
|
|
"scaleDistribution":{"type":"linear"},
|
|
"showPoints":"never",
|
|
"spanNulls":true,
|
|
"stacking":{"group":"A","mode":"none"},
|
|
"thresholdsStyle":{"mode":"area"}
|
|
},
|
|
"decimals":0,
|
|
"links":[{"title":"PGSQL Node: Space Exhaust for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=51&${__url_time_range}"}],
|
|
"mappings":[{"options":{"-1":{"index":1,"text":"+Inf"}},"type":"value"},{"options":{"match":"null+nan","result":{"index":0,"text":"N/A"}},"type":"special"}],
|
|
"max":3200000,
|
|
"min":0,
|
|
"thresholds":{"mode":"absolute","steps":[{"color":"text"},{"color":"purple","value":3600},{"color":"orange","value":86400},{"color":"yellow","value":604800},{"color":"#346f36cc","value":126230400}]},
|
|
"unit":"s"
|
|
},
|
|
"overrides":[]
|
|
},
|
|
"gridPos":{"h":7,"w":24,"x":0,"y":99},
|
|
"id":78,
|
|
"options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
|
|
"pluginVersion":"8.0.0",
|
|
"targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"node:fs:space_exhaust{}","interval":"","legendFormat":"{{ ip }}:{{ mountpoint }}","refId":"A"}],
|
|
"title":"NodeFsExhaust",
|
|
"type":"timeseries"
|
|
},{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"fieldConfig":{
|
|
"defaults":{
|
|
"color":{"mode":"palette-classic"},
|
|
"custom":{
|
|
"axisCenteredZero":false,
|
|
"axisColorMode":"text",
|
|
"axisLabel":"",
|
|
"axisPlacement":"auto",
|
|
"axisSoftMax":1,
|
|
"axisSoftMin":0,
|
|
"barAlignment":0,
|
|
"drawStyle":"line",
|
|
"fillOpacity":0,
|
|
"gradientMode":"none",
|
|
"hideFrom":{"legend":false,"tooltip":false,"viz":false},
|
|
"lineInterpolation":"linear",
|
|
"lineWidth":1,
|
|
"pointSize":5,
|
|
"scaleDistribution":{"type":"linear"},
|
|
"showPoints":"auto",
|
|
"spanNulls":false,
|
|
"stacking":{"group":"A","mode":"none"},
|
|
"thresholdsStyle":{"mode":"area"}
|
|
},
|
|
"links":[{"title":"PGSQL Node: iNode Usage for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=174&${__url_time_range}"}],
|
|
"mappings":[],
|
|
"max":3,
|
|
"min":0,
|
|
"thresholds":{"mode":"absolute","steps":[{"color":"#3e668f"},{"color":"green","value":0.1},{"color":"yellow","value":0.8},{"color":"orange","value":0.9},{"color":"red","value":0.95},{"color":"purple","value":0.99},{"color":"text","value":1}]},
|
|
"unit":"percentunit"
|
|
},
|
|
"overrides":[]
|
|
},
|
|
"gridPos":{"h":7,"w":24,"x":0,"y":106},
|
|
"id":58,
|
|
"options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
|
|
"targets":[{"exemplar":false,"expr":"node:fs:inode_usage","interval":"","legendFormat":"{{ ins }}:{{ mountpoint }}","refId":"A"}],
|
|
"title":"NodeFsFilesFull",
|
|
"type":"timeseries"
|
|
},{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"fieldConfig":{
|
|
"defaults":{
|
|
"color":{"mode":"palette-classic"},
|
|
"custom":{
|
|
"axisCenteredZero":false,
|
|
"axisColorMode":"text",
|
|
"axisLabel":"",
|
|
"axisPlacement":"auto",
|
|
"axisSoftMax":1,
|
|
"axisSoftMin":0,
|
|
"barAlignment":0,
|
|
"drawStyle":"line",
|
|
"fillOpacity":0,
|
|
"gradientMode":"none",
|
|
"hideFrom":{"legend":false,"tooltip":false,"viz":false},
|
|
"lineInterpolation":"linear",
|
|
"lineWidth":1,
|
|
"pointSize":5,
|
|
"scaleDistribution":{"type":"linear"},
|
|
"showPoints":"auto",
|
|
"spanNulls":false,
|
|
"stacking":{"group":"A","mode":"none"},
|
|
"thresholdsStyle":{"mode":"area"}
|
|
},
|
|
"links":[{"title":"PGSQL Node: FD for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=58&${__url_time_range}"}],
|
|
"mappings":[],
|
|
"max":3,
|
|
"min":0,
|
|
"thresholds":{"mode":"absolute","steps":[{"color":"#3e668f"},{"color":"green","value":0.1},{"color":"orange","value":0.8},{"color":"orange","value":0.9},{"color":"red","value":0.95},{"color":"purple","value":0.99},{"color":"text","value":1}]},
|
|
"unit":"percentunit"
|
|
},
|
|
"overrides":[]
|
|
},
|
|
"gridPos":{"h":7,"w":24,"x":0,"y":113},
|
|
"id":59,
|
|
"options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
|
|
"targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"node:ins:fd_usage","interval":"","legendFormat":"{{ ip }}","refId":"A"}],
|
|
"title":"NodeFdFull",
|
|
"type":"timeseries"
|
|
}
|
|
],
|
|
"title":"NODE: Filesystem",
|
|
"type":"row"
|
|
},
|
|
{
|
|
"collapsed":true,
|
|
"gridPos":{"h":1,"w":24,"x":0,"y":41},
|
|
"id":62,
|
|
"panels":[
|
|
{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"description":"TCP Error Rate > 1 / s",
|
|
"fieldConfig":{
|
|
"defaults":{
|
|
"color":{"mode":"palette-classic"},
|
|
"custom":{
|
|
"axisCenteredZero":false,
|
|
"axisColorMode":"text",
|
|
"axisLabel":"",
|
|
"axisPlacement":"auto",
|
|
"axisSoftMin":0,
|
|
"barAlignment":0,
|
|
"drawStyle":"line",
|
|
"fillOpacity":0,
|
|
"gradientMode":"none",
|
|
"hideFrom":{"legend":false,"tooltip":false,"viz":false},
|
|
"lineInterpolation":"linear",
|
|
"lineWidth":1,
|
|
"pointSize":5,
|
|
"scaleDistribution":{"type":"linear"},
|
|
"showPoints":"auto",
|
|
"spanNulls":false,
|
|
"stacking":{"group":"A","mode":"none"},
|
|
"thresholdsStyle":{"mode":"area"}
|
|
},
|
|
"decimals":1,
|
|
"links":[{"title":"PGSQL Node: TCP Errors for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=55&${__url_time_range}"}],
|
|
"mappings":[],
|
|
"max":1,
|
|
"min":0,
|
|
"thresholds":{"mode":"absolute","steps":[{"color":"green"},{"color":"orange","value":1},{"color":"red","value":2}]},
|
|
"unit":"cps"
|
|
},
|
|
"overrides":[]
|
|
},
|
|
"gridPos":{"h":7,"w":24,"x":0,"y":121},
|
|
"id":71,
|
|
"options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
|
|
"targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"rate(node:ins:tcp_error[1m])","interval":"","legendFormat":"{{ ip }}","refId":"A"}],
|
|
"title":"NodeTcpErrHigh",
|
|
"type":"timeseries"
|
|
},{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"fieldConfig":{
|
|
"defaults":{
|
|
"color":{"mode":"palette-classic"},
|
|
"custom":{
|
|
"axisCenteredZero":false,
|
|
"axisColorMode":"text",
|
|
"axisLabel":"",
|
|
"axisPlacement":"auto",
|
|
"axisSoftMax":1,
|
|
"axisSoftMin":0,
|
|
"barAlignment":0,
|
|
"drawStyle":"line",
|
|
"fillOpacity":0,
|
|
"gradientMode":"none",
|
|
"hideFrom":{"legend":false,"tooltip":false,"viz":false},
|
|
"lineInterpolation":"linear",
|
|
"lineWidth":1,
|
|
"pointSize":5,
|
|
"scaleDistribution":{"log":10,"type":"log"},
|
|
"showPoints":"auto",
|
|
"spanNulls":false,
|
|
"stacking":{"group":"A","mode":"none"},
|
|
"thresholdsStyle":{"mode":"area"}
|
|
},
|
|
"links":[{"title":"PGSQL Node: TCP Retransmit for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=52&${__url_time_range}"}],
|
|
"mappings":[],
|
|
"min":0,
|
|
"thresholds":{"mode":"absolute","steps":[{"color":"#3e668f"},{"color":"green","value":1e-06},{"color":"yellow","value":0.0001},{"color":"orange","value":0.001},{"color":"red","value":0.01},{"color":"purple","value":0.1}]},
|
|
"unit":"sci"
|
|
},
|
|
"overrides":[]
|
|
},
|
|
"gridPos":{"h":9,"w":24,"x":0,"y":128},
|
|
"id":70,
|
|
"options":{"legend":{"calcs":["max"],"displayMode":"table","placement":"right","showLegend":true},"tooltip":{"mode":"multi","sort":"none"}},
|
|
"targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"node:ins:tcp_retrans_ratio1m","interval":"","legendFormat":"{{ ip }}","refId":"A"}],
|
|
"title":"NodeTcpRetransHigh",
|
|
"type":"timeseries"
|
|
}
|
|
],
|
|
"title":"Node: Network",
|
|
"type":"row"
|
|
},
|
|
{
|
|
"collapsed":true,
|
|
"gridPos":{"h":1,"w":24,"x":0,"y":42},
|
|
"id":25,
|
|
"panels":[
|
|
{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"description":"Check ntp protocol / clock sanity",
|
|
"fieldConfig":{
|
|
"defaults":{
|
|
"color":{"mode":"palette-classic"},
|
|
"custom":{
|
|
"axisCenteredZero":false,
|
|
"axisColorMode":"text",
|
|
"axisLabel":"",
|
|
"axisPlacement":"hidden",
|
|
"barAlignment":0,
|
|
"drawStyle":"line",
|
|
"fillOpacity":0,
|
|
"gradientMode":"none",
|
|
"hideFrom":{"legend":false,"tooltip":false,"viz":false},
|
|
"insertNulls":false,
|
|
"lineInterpolation":"linear",
|
|
"lineWidth":1,
|
|
"pointSize":5,
|
|
"scaleDistribution":{"type":"linear"},
|
|
"showPoints":"auto",
|
|
"spanNulls":false,
|
|
"stacking":{"group":"A","mode":"none"},
|
|
"thresholdsStyle":{"mode":"area"}
|
|
},
|
|
"links":[{"title":"PGSQL Node: NTP Sanity for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=70&${__url_time_range}"}],
|
|
"mappings":[],
|
|
"max":1.1,
|
|
"min":0,
|
|
"thresholds":{"mode":"absolute","steps":[{"color":"red"},{"color":"green","value":0.5}]},
|
|
"unit":"none"
|
|
},
|
|
"overrides":[]
|
|
},
|
|
"gridPos":{"h":7,"w":24,"x":0,"y":51},
|
|
"id":75,
|
|
"options":{"legend":{"calcs":["min"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Min","sortDesc":false},"tooltip":{"mode":"single","sort":"none"}},
|
|
"targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"editorMode":"code","exemplar":false,"expr":"min without() (node_timex_sync_status)","interval":"","legendFormat":"{{ ip }}","range":true,"refId":"A"}],
|
|
"title":"NodeTimeSync",
|
|
"type":"timeseries"
|
|
},{
|
|
"datasource":{"type":"prometheus","uid":"ds-prometheus"},
|
|
"description":"Clock time offset alert > 64ms",
|
|
"fieldConfig":{
|
|
"defaults":{
|
|
"color":{"mode":"palette-classic"},
|
|
"custom":{
|
|
"axisCenteredZero":false,
|
|
"axisColorMode":"text",
|
|
"axisLabel":"",
|
|
"axisPlacement":"auto",
|
|
"barAlignment":0,
|
|
"drawStyle":"line",
|
|
"fillOpacity":0,
|
|
"gradientMode":"none",
|
|
"hideFrom":{"legend":false,"tooltip":false,"viz":false},
|
|
"insertNulls":false,
|
|
"lineInterpolation":"linear",
|
|
"lineWidth":1,
|
|
"pointSize":5,
|
|
"scaleDistribution":{"log":2,"type":"log"},
|
|
"showPoints":"auto",
|
|
"spanNulls":false,
|
|
"stacking":{"group":"A","mode":"none"},
|
|
"thresholdsStyle":{"mode":"area"}
|
|
},
|
|
"links":[{"title":"PGSQL Node: Time Offset for ${__field.labels.ins}","url":"/d/node-instance?var-ins=${__field.labels.ins}&viewPanel=83&${__url_time_range}"}],
|
|
"mappings":[],
|
|
"thresholds":{"mode":"absolute","steps":[{"color":"#3e668f"},{"color":"green","value":0.001},{"color":"yellow","value":0.01},{"color":"orange","value":0.064},{"color":"red","value":1},{"color":"purple","value":60},{"color":"text","value":600}]},
|
|
"unit":"s"
|
|
},
|
|
"overrides":[]
|
|
},
|
|
"gridPos":{"h":7,"w":24,"x":0,"y":58},
|
|
"id":74,
|
|
"options":{"legend":{"calcs":["max"],"displayMode":"table","placement":"right","showLegend":true,"sortBy":"Max","sortDesc":true},"tooltip":{"mode":"single","sort":"none"}},
|
|
"targets":[{"datasource":{"type":"prometheus","uid":"ds-prometheus"},"exemplar":false,"expr":"abs(node_timex_offset_seconds)","interval":"","legendFormat":"{{ ip }}","refId":"A"}],
|
|
"title":"NodeClockSkew",
|
|
"type":"timeseries"
|
|
}
|
|
],
|
|
"title":"Node: Time",
|
|
"type":"row"
|
|
}
|
|
],
|
|
"preload":false,
|
|
"refresh":"",
|
|
"schemaVersion":42,
|
|
"tags":["Pigsty","NODE","Overview","Alert"],
|
|
"templating":{"list":[]},
|
|
"time":{"from":"now-1h","to":"now"},
|
|
"timepicker":{"refresh_intervals":["2s","5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"]},
|
|
"timezone":"browser",
|
|
"title":"Node Alert",
|
|
"uid":"node-alert",
|
|
"version":1,
|
|
"weekStart":""
|
|
} |